Wave MR Scene And Anchor Native Development Guide

Overview

To develop MR applications using Wave SDK, please refer to the following documents:

Contents

Note

  • You can download the Wave SDK from here. And check Native MR sample in SDK/samples/wvr_native_anchor.
  • To build the project, please reference Native SDK Getting Started.

1. Permission & Manifest

In order to use scene perception API, developer should add the feature in AndroidManifest.xml. If developer wants to use scene mesh API, permission is also needed.

//andoirManifest.xml
//scene perception
<uses-feature android:name="wave.feature.sceneperception" android:required="true"/>
//scene mesh
<uses-permission android:name="wave.permission.GET_SCENE_MESH"/>

//MainActivity.java
int hasPermission=checkSelfPermission(SCENE_PERMISSION);
if(hasPermission!=PackageManager.PERMISSION_GRANTED){
    Log.i(TAG, "Permission not granted try to aquire: "+SCENE_PERMISSION);
    requestPermissions(new String[]{SCENE_PERMISSION},REQUEST_CODE);
}
@Override
public void onRequestPermissionsResult(int requestCode,  String[] permissions,  int[] grantResults) {
    if (requestCode == REQUEST_CODE && grantResults != null && grantResults.length > 0) {
        if (grantResults[0] == PackageManager.PERMISSION_GRANTED) {
            mHasPermission = true;
            Log.i(TAG, "Permission granted");
        }
        else{
            Log.i(TAG, "Permission not granted");
        }
    }
}

2. Show passthrouh

Show Passthrough underlay by calling WVR_ShowPassthroughUnderlay() after WVR_Init(). User can also call WVR_SetPassthroughImageFocus(WVR_PassthroughImageFocus_Scale) to have better MR alignment.

static bool showUnderlay = true;
WVR_ShowPassthroughUnderlay(showUnderlay);
WVR_SetPassthroughImageFocus(WVR_PassthroughImageFocus_Scale);
_images/passThrough.png

3. Anchor

3-1. Init

In order to use anchor API, developer should call WVR_StartScene() before calling any anchor API.

WVR_Result result = WVR_StartScene();
if (result != WVR_Success)
{
    LOGE("WVR_StartScene init fail %d", result);
    return false;
}

3-2. Spatial Anchors

Spatial anchor will be destroyed after closing app.

3-2-1. Create spatial anchors - WVR_CreateSpatialAnchor()

Application can create a spatial anchor by calling WVR_CreateSpatialAnchor(). The created spatial anchor will be recorded by the device service. After the device service records the added spatial anchor successfully, the device service will send WVR_EventType_SpatialAnchor_Changed event to WVR_Event_t. Note that the last char of WVR_SpatialAnchorCreateInfo.anchorName.name must be an ending char.

// this example anchor name should be "s/c/p=time=scale.x=scale.y=scale.z="
WVR_SpatialAnchorCreateInfo info;
info.originModel = WVR_PoseOriginModel_OriginOnHead;
info.pose = convertToWVRPose(pose);

// anchor name()
std::string dash("=");
// type tag
std::string spatialName;
switch (type)
{
case spatial:
    // represent spatial anchor
    spatialName = {"s"};
    break;
case cached:
    // represent cached anchor
    spatialName = {"c"};
    break;
case persisted:
    // represent persisted anchor
    spatialName = {"p"};
    break;
default:
    LOGE("createCubeAnchor type get error");
    break;
}
spatialName += dash;

// time tag
static int preventSameName = 0;
preventSameName = (preventSameName + 1) % 10; // this allow create 10 times in one second
uint64_t sec = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();
std::string tempString = std::to_string(sec);
tempString = std::to_string(preventSameName) + tempString;
spatialName += tempString;
spatialName += dash;

// scale tag
spatialName += std::to_string(round(scale.x * 10000) / 10000);
spatialName += dash;
spatialName += std::to_string(round(scale.y * 10000) / 10000);
spatialName += dash;
spatialName += std::to_string(round(scale.z * 10000) / 10000);
spatialName += dash;

// add ending char
spatialName.push_back('\0');

std::copy(spatialName.begin(), spatialName.end(), info.anchorName.name);
LOGI("addAnchor , scale:(%f, %f, %f)", round(scale.x * 10000) / 10000, round(scale.y * 10000) / 10000, round(scale.z * 10000) / 10000);

// create anchor
WVR_SpatialAnchor anchor;
LOGI("create spatial anchor name(%d)=%s", spatialName.size(), spatialName.c_str());
switch (type)
{
case spatial:
    // create spatical anchor
    if (WVR_CreateSpatialAnchor(&info, &anchor) != WVR_Success)
    {
        LOGE("create spatial anchor fail");
    }
    break;
case cached:
    if (WVR_CreateSpatialAnchor(&info, &anchor) != WVR_Success)
    {
        LOGE("create spatial anchor fail");
        return false;
    }
    // create cached anchor
    WVR_SpatialAnchorCacheInfo cacheInfo;
    cacheInfo.spatialAnchor = anchor;
    cacheInfo.cachedSpatialAnchorName = info.anchorName;
    LOGI("create cached anchor");
    if (WVR_CacheSpatialAnchor(&cacheInfo) != WVR_Success)
        LOGE("create cached anchor fail");

    /*not necessary on every app*/
    // destroy spatial anchor. In this sample, we will refrsh all spatial anchor by
    // using WVR_EnumerateCachedSpatialAnchorNames and WVR_CreateSpatialAnchorFromCacheName
    WVR_DestroySpatialAnchor(anchor);
    break;
case persisted:
    // check if persisted anchor is full
    WVR_PersistedSpatialAnchorCountGetInfo countInfo;
    WVR_GetPersistedSpatialAnchorCount(&countInfo);
    if (countInfo.currentTrackingCount < countInfo.maximumTrackingCount)
    {
        if (WVR_CreateSpatialAnchor(&info, &anchor) != WVR_Success)
        {
            LOGE("create spatial anchor fail");
            return false;
        }
        // create persisted anchor
        WVR_SpatialAnchorPersistInfo persistInfo;
        persistInfo.spatialAnchor = anchor;
        persistInfo.persistedSpatialAnchorName = info.anchorName;
        LOGI("create persisted anchor");
        if (WVR_PersistSpatialAnchor(&persistInfo) != WVR_Success)
            LOGE("create persisted anchor fail");

        /*not necessary on every app*/
        // destroy spatial anchor. In this sample, we will refrsh all spatial anchor by
        // using WVR_EnumerateCachedSpatialAnchorNames and WVR_CreateSpatialAnchorFromCacheName
        WVR_DestroySpatialAnchor(anchor);
    }
    else
    {
        LOGE("persist anchor is full");
    }
    break;
default:
    LOGE("type get error");
    break;
}

3-2-2. Get the existing spatial anchor - WVR_EnumerateSpatialAnchors()

Application should call WVR_EnumerateSpatialAnchors() twice to get the number of the existing spatial anchors and WVR_SpatialAnchor object. Application should implement this process in onCreate and onResume to get WVR_EventType_SpatialAnchor_Changed event.

WVR_SpatialAnchor *allSpatialAnchor;
uint32_t allSpatialAnchorCount;

while (WVR_PollEventQueue(&event))
{
    ...
    // anchor event
    if (event.common.type == WVR_EventType_SpatialAnchor_Changed)
    {
        LOGI("receive WVR_EventType_SpatialAnchor_Changed event");
        mAnchorManager->getExistedSpatialAnchor(mCubes);
    }
    ...
}

bool anchorManager::getExistedSpatialAnchor(CubeManager *cm)
{
    // WVR API get all spatial anchor num and allocate memory to get data
    allSpatialAnchorCount = 0;
    WVR_EnumerateSpatialAnchors(0, &allSpatialAnchorCount, nullptr);
    LOGI("get all spatial anchor anchorCount=%u", allSpatialAnchorCount);

    // clear memory
    if (allSpatialAnchor != nullptr)
        free(allSpatialAnchor);
    cm->clearAllTargetCubes();

    // assign memory space for spatial anchor
    allSpatialAnchor = (WVR_SpatialAnchor *)malloc(sizeof(WVR_SpatialAnchor) * allSpatialAnchorCount);
    WVR_EnumerateSpatialAnchors(allSpatialAnchorCount, &allSpatialAnchorCount, allSpatialAnchor);

    // classified different spatial anchors
    for (int a = 0; a < allSpatialAnchorCount; a++)
    {
        Matrix4 pose;
        Vector3 scale;
        WVR_SpatialAnchorState anchorState;
        WVR_GetSpatialAnchorState(allSpatialAnchor[a], WVR_PoseOriginModel_OriginOnHead, &anchorState);
        std::string mName(anchorState.anchorName.name);

        switch (mName[0])
        {
        // pure spatial anchor
        case 's':
            pose = convertToMatrix4(anchorState.pose);
            scale = getScaleFromName(mName);
            cm->addCubeFromAnchor(spatial, a, pose, scale, anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking ? false : true);
            break;
        // spatial anchor which has been cached
        case 'c':
            pose = convertToMatrix4(anchorState.pose);
            scale = getScaleFromName(mName);
            cm->addCubeFromAnchor(cached, a, pose, scale, anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking ? false : true);
            break;
        // spatial anchor which has been persisted
        case 'p':
            pose = convertToMatrix4(anchorState.pose);
            scale = getScaleFromName(mName);
            cm->addCubeFromAnchor(persisted, a, pose, scale, anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking ? false : true);
            break;
        default:
            LOGE("Wrong anchor name exist: %s", mName.c_str());
            break;
        }
    }
    return true;
}

3-2-3. Get the existing spatial anchor data - WVR_GetSpatialAnchorState()

Application can send WVR_SpatialAnchor object to WVR_GetSpatialAnchorState() to update the anchor status which includes pose, name and tracking state. If the anchor’s trackingState is not equal to WVR_SpatialAnchorTrackingState_Tracking, developer should not use this anchor pose to update model pose.

// WVR API get all spatial anchor num and allocate memory to get data
allSpatialAnchorCount = 0;
WVR_EnumerateSpatialAnchors(0, &allSpatialAnchorCount, nullptr);
LOGI("get all spatial anchor anchorCount=%u", allSpatialAnchorCount);

// clear memory
if (allSpatialAnchor != nullptr)
    free(allSpatialAnchor);
cm->clearAllTargetCubes();

// assign memory space for anchor
allSpatialAnchor = (WVR_SpatialAnchor *)malloc(sizeof(WVR_SpatialAnchor) * allSpatialAnchorCount);
WVR_EnumerateSpatialAnchors(allSpatialAnchorCount, &allSpatialAnchorCount, allSpatialAnchor);

// classified different spatial anchors
for (int a = 0; a < allSpatialAnchorCount; a++)
{
    Matrix4 pose;
    Vector3 scale;
    WVR_SpatialAnchorState anchorState;
    WVR_GetSpatialAnchorState(allSpatialAnchor[a], WVR_PoseOriginModel_OriginOnHead, &anchorState);
    std::string mName(anchorState.anchorName.name);

    switch (mName[0])
    {
    // update spatial anchor pose to render
    case 's':
        pose = convertToMatrix4(anchorState.pose);
        scale = getScaleFromName(mName);
        cm->addCubeFromAnchor(spatial, a, pose, scale, anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking ? false : true);
        break;
    // update catched anchor pose to render
    case 'c':
        pose = convertToMatrix4(anchorState.pose);
        scale = getScaleFromName(mName);
        cm->addCubeFromAnchor(cached, a, pose, scale, anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking ? false : true);
        break;
    // update persisted anchor pose to render
    case 'p':
        pose = convertToMatrix4(anchorState.pose);
        scale = getScaleFromName(mName);
        cm->addCubeFromAnchor(persisted, a, pose, scale, anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking ? false : true);
        break;
    default:
        LOGE("Wrong anchor name exist: %s", mName.c_str());
        break;
    }
}

3-2-4. Delete spatial anchor - WVR_DestroySpatialAnchor()

Application can delete spatial anchors by calling WVR_DestroySpatialAnchor(). Note that the device service sends WVR_EventType_SpatialAnchor_Changed after removing this spatial anchor.

// delete spatial anchor
WVR_DestroySpatialAnchor(*(allSpatialAnchor + cm->leftTouchVector[a].anchorIndex))
// will get WVR_EventType_SpatialAnchor_Changed event soon

3-3. Cached Anchors

Cached anchor will be destroyed when device can not recognize the surrounding area.

3-3-1. Cache spatial anchor - WVR_CacheSpatialAnchor()

Application can cache spatial anchor by calling WVR_CacheSpatialAnchor(). After the device service records a anchor is cached successfully, it will send WVR_EventType_CachedAnchor_Changed event to WVR_Event_t. Note that, in order to cache anchor successfully, WVR_SpatialAnchorCacheInfo.cachedSpatialAnchorName must be a unique name.

code as 3-2-1

3-3-2. Get the existing cached anchor - WVR_EnumerateCachedSpatialAnchorNames()

Application should call WVR_EnumerateCachedSpatialAnchorNames() twice to get the number of the cached anchors and their names. Application should implement this process in onCreate and onResume to get WVR_EventType_CachedAnchor_Changed event.

uint32_t cachedAnchorCount;

while (WVR_PollEventQueue(&event))
{
    ...
    // anchor event
    if (event.common.type == WVR_EventType_CachedAnchor_Changed)
    {
        LOGI("receive WVR_EventType_CachedAnchor_Changed event");
        mAnchorManager->getExistedCachedAnchor();
    }
    ...
}

bool anchorManager::getExistedCachedAnchor()
{
    // get cache anchor name
    cachedAnchorCount = 0;
    WVR_EnumerateCachedSpatialAnchorNames(0, &cachedAnchorCount, nullptr);
    LOGI("Enumerate cached count=%d", cachedAnchorCount);
    WVR_SpatialAnchorName *allCachedName;
    allCachedName = (WVR_SpatialAnchorName *)malloc(sizeof(WVR_SpatialAnchorName) * cachedAnchorCount);
    WVR_EnumerateCachedSpatialAnchorNames(cachedAnchorCount, &cachedAnchorCount, allCachedName);
    // Let all cached anchor as spatial anchor
    WVR_SpatialAnchorFromCacheNameCreateInfo cacheInfo;
    for (int a = 0; a < cachedAnchorCount; a++)
    {
        cacheInfo.cachedSpatialAnchorName = allCachedName[a];
        cacheInfo.spatialAnchorName = allCachedName[a];
        WVR_SpatialAnchor *ptr; // will get this anchor address later(via WVR_EnumerateSpatialAnchors)
        LOGI("create spatial anchor from cached anchor");
        if (WVR_CreateSpatialAnchorFromCacheName(&cacheInfo, ptr) != WVR_Success)
            LOGI("already have same cached anchor as spatial anchor");
    }
    free(allCachedName);
    return true;
}

3-3-3. Create/restore the spatial anchor by the cached name - WVR_CreateSpatialAnchorFromCacheName()

Application can also use cached anchor name to create and restore the spatial anchor. After device service records added spatial anchor successfully, it will send WVR_EventType_SpatialAnchor_Changed event to WVR_Event_t.

code as 3-3-2

3-3-4. uncache cached anchor - WVR_UncacheSpatialAnchor()/WVR_ClearCachedSpatialAnchors()

Application can call WVR_UncacheSpatialAnchor() to uncache anchor, or call WVR_ClearCachedSpatialAnchors() to uncache all cached anchor in the device service. After the device service records a cached anchor is uncached successfully, it will send WVR_EventType_CachedAnchor_Changed event to WVR_Event_t.

...
/*uncache one cached anchor*/
WVR_SpatialAnchorState anchorState;
WVR_GetSpatialAnchorState(allSpatialAnchor[cm->leftTouchVector[a].anchorIndex], WVR_PoseOriginModel_OriginOnHead, &anchorState);
if (type == cached)
{
    // in this sample, cachedSpatialAnchorName equal to spatialAnchorName
    // Therefore, we can uncache anchor via spatialAnchorName
    WVR_UncacheSpatialAnchor(&anchorState.anchorName);
}
...

/*uncache all cached anchor*/
WVR_ClearCachedSpatialAnchors()

3-3-5. Get the existing cached anchor data - WVR_GetSpatialAnchorState()

In order to get cached anchor pose, application should update its status.

code as 3-2-3

3-4. Persisted Anchors

Persisted anchor can be shared betweem devices.

3-4-1. Persisted spatial anchor - WVR_PersistSpatialAnchor()

Application should create a persisted spatial anchor by calling WVR_PersistSpatialAnchor() after checking the maximum number by WVR_PersistedSpatialAnchorCountGetInfo.maximumTrackingCount. After the device service records a persisted anchor is added successfully, it will send WVR_EventType_PersistedAnchor_Changed event to WVR_Event_t. Note that to create a persisted anchor successfully, WVR_SpatialAnchorPersistInfo.persistedSpatialAnchorName should be a unique name.

code as 3-2-1

3-4-2. Get the existing persisted anchor - WVR_EnumeratePersistedSpatialAnchorNames()

Application should call WVR_EnumeratePersistedSpatialAnchorNames() twice to get the number of the persisted anchors and their name. Application should implement this process in onCreate and onResume to get WVR_EventType_PersistedAnchor_Changed event.

uint32_t cachedAnchorCount;

while (WVR_PollEventQueue(&event))
{
    ...
    // anchor event
    if (event.common.type == WVR_EventType_PersistedAnchor_Changed)
    {
        LOGI("receive WVR_EventType_PersistedAnchor_Changed event");
        mAnchorManager->getExistedPersistedAnchor();
    }
    ...
}

bool anchorManager::getExistedPersistedAnchor()
{
    // get persisted anchor name
    persistedAnchorCount = 0;
    WVR_EnumeratePersistedSpatialAnchorNames(0, &persistedAnchorCount, nullptr);
    LOGI("Enumerate persisted count=%d", persistedAnchorCount);
    WVR_SpatialAnchorName *allPersistedName;
    allPersistedName = (WVR_SpatialAnchorName *)malloc(sizeof(WVR_SpatialAnchorName) * persistedAnchorCount);
    WVR_EnumeratePersistedSpatialAnchorNames(persistedAnchorCount, &persistedAnchorCount, allPersistedName);
    // restore spatial anchor via persisted anchor name
    WVR_SpatialAnchorFromPersistenceNameCreateInfo persistInfo;
    for (int a = 0; a < persistedAnchorCount; a++)
    {
        persistInfo.persistedSpatialAnchorName = allPersistedName[a];
        persistInfo.spatialAnchorName = allPersistedName[a];
        WVR_SpatialAnchor *ptr; // will get this anchor address later(via WVR_EnumerateSpatialAnchors)
        LOGI("create spatial anchor from persisted anchor");
        if (WVR_CreateSpatialAnchorFromPersistenceName(&persistInfo, ptr) != WVR_Success)
            LOGI("already have same persisted anchor as spatial anchor");
    }
    free(allPersistedName);
    return true;
}

3-4-3. Create/restore the spatial anchor by persisted anchor name - WVR_CreateSpatialAnchorFromPersistenceName()

Application can also use persisted anchor name to create or restore the spatial anchor. After the device service records a spatial anchor is added successfully, it will send WVR_EventType_SpatialAnchor_Changed event to WVR_Event_t.

code as 3-4-2

3-4-4. Unpersist persisted anchor - WVR_UnpersistSpatialAnchor()/WVR_ClearPersistedSpatialAnchors()

Application can call WVR_UnpersistSpatialAnchor() to unpersist one persisted anchor, or call WVR_ClearPersistedSpatialAnchors() to unpersist all existing persisted anchors in the device service. After the device service records an unpersist anchor is cached successfully, it will send WVR_EventType_PersistedAnchor_Changed event to WVR_Event_t.

...
/*uncache one cached anchor*/
WVR_SpatialAnchorState anchorState;
WVR_GetSpatialAnchorState(allSpatialAnchor[cm->leftTouchVector[a].anchorIndex], WVR_PoseOriginModel_OriginOnHead, &anchorState);
if (type == persisted)
{
    // in this sample, persistedSpatialAnchorName equal to spatialAnchorName
    // Therefore, we can unpersist anchor via spatialAnchorName
    WVR_UnpersistSpatialAnchor(&anchorState.anchorName);
}

...
/*unpersist all persisted anchor*/
WVR_ClearPersistedSpatialAnchors()

3-4-5. Get the existing persisted anchor data - WVR_GetSpatialAnchorState()

In order to get persisted anchor pose, application should update its status.

code as 3-2-3

3-4-6. Export the persisted anchor data - WVR_ExportPersistedSpatialAnchor()

Persisted anchor can be shared between different devices. In order to share persisted anchor data, application should call WVR_ExportPersistedSpatialAnchor() twice to get the size and data of the persisted anchor. Note that this API is always called in the background thread because it may block the thread for a log time.

// do this work in background thread
if (taskPtr->workType == EXPORT_ANCHOR)
{
    LOGI("BGThread EXPORT_ANCHOR...");
    if (taskPtr->persistData != nullptr)
    {
        free(taskPtr->persistData);
        taskPtr->persistDataSize = 0;
    }

    if (WVR_ExportPersistedSpatialAnchor(&taskPtr->persistedName, 0, &taskPtr->persistDataSize, nullptr) == WVR_Success)
    {
        LOGI("export persisted anchor persistDataSize=%d", taskPtr->persistDataSize);
        taskPtr->persistData = (char *)malloc(sizeof(char) * taskPtr->persistDataSize);
    }

    if (WVR_ExportPersistedSpatialAnchor(&taskPtr->persistedName, taskPtr->persistDataSize, &taskPtr->persistDataSize, taskPtr->persistData) != WVR_Success)
    {
        LOGE("export persisted anchor fail");
        free(taskPtr->persistData);
        taskPtr->persistDataSize = 0;
    }
    else
    {
        // store persisted anchor in device. Can push this file to other device
        std::ofstream binFile("/data/user/0/com.htc.vr.samples.wvr_native_anchor/files/export.bin", std::ios::out | std::ios::binary);
        binFile.write(taskPtr->persistData, taskPtr->persistDataSize);
        binFile.close();
    }
    LOGI("BGThread EXPORT_ANCHOR done");
}

3-4-7. Import persisted anchor data - WVR_ImportPersistedSpatialAnchor()

Persisted anchor can be shared between different devices. In order to get the persisted anchors from another devices, application should call WVR_ImportPersistedSpatialAnchor() to import the persisted anchors. After the device service records a persisted anchor is imported successfully, it send WVR_EventType_PersistedAnchor_Changed event to WVR_Event_t. Note that this API is always be called in background thread because it may block the thread for quite a log time.

if (taskPtr->workType == IMPORT_ANCHOR)
{
    LOGI("BGThread IMPORT_ANCHOR...");
    // open persisted anchor in device. This file is pushed by other device
    std::ifstream importFile("/data/user/0/com.htc.vr.samples.wvr_native_anchor/files/import.bin", std::ios::in);
    if (importFile.is_open())
    {
        LOGI("import anchor file exist, replace import data");
        if (taskPtr->persistData != nullptr)
        {
            free(taskPtr->persistData);
            taskPtr->persistDataSize = 0;
        }

        importFile.seekg(0, std::ifstream::end);
        taskPtr->persistDataSize = importFile.tellg();
        LOGI("import persisted anchor Size=%d", taskPtr->persistDataSize);
        importFile.seekg(0, std::ifstream::beg);
        taskPtr->persistData = (char *)malloc(sizeof(char) * taskPtr->persistDataSize);
        importFile.read(taskPtr->persistData, taskPtr->persistDataSize);
        importFile.close();
    }

    if (taskPtr->persistDataSize > 0)
    {
        WVR_ImportPersistedSpatialAnchor(taskPtr->persistDataSize, taskPtr->persistData);
    }
    else
    {
        LOGW("no persisted data can be import");
    }
    LOGI("BGThread IMPORT_ANCHOR done");
}

3-5. Render anchor object

In order to render a model with anchor, developer should use the anchor’s pose as the model pose matrix and sent it to OpenGL shader program.

//update anchor pose
WVR_GetSpatialAnchorState(mAnchor[cm->cubeVector[a]->anchorIndex], WVR_PoseOriginModel_OriginOnHead, &anchorState);
if (anchorState.trackingState == WVR_SpatialAnchorTrackingState_Tracking)
{
    allAnchorCanTrack = true;
    pose = convertToMatrix4(anchorState.pose);
    cm->cubeVector[a]->mPose = pose;
    cm->cubeVector[a]->anchorLost = false;
}
else
{
    cm->cubeVector[a]->anchorLost = true;
    LOGI("Anchor cubeVector(%d) lost tracking", a);
}

//render
Matrix4 matrix = projection * eye * view * cubeVector[a]->mPose;
glUniformMatrix4fv(mMatrixLocation, 1, false, matrix.get());
glUniform3f(mScale, cubeVector[a]->mScale.x, cubeVector[a]->mScale.y, cubeVector[a]->mScale.z);

mVAO->bindVAO();
glActiveTexture(GL_TEXTURE0);
mTexture->bindTexture();
glDrawElements(GL_TRIANGLES, singleIndexArray.size(), GL_UNSIGNED_INT, 0);
mTexture->unbindTexture();
mVAO->unbindVAO();
_images/renderAnchor.png

4. Scene Plane & Plane Mesh

4-1. Init

In order to use plane API, developer should call WVR_StartScene() and WVR_StartScenePerception() before calling any plane API.

// Loading the WVR scene
WVR_Result result = WVR_StartScene();
if (result != WVR_Success)
{
    LOGE("WVR_StartScene init fail %d", result);
    return false;
}

// start perception before get scene plane (need permission)
// can call WVR_StopScenePerception when you won't need to update data
// WVR_StopScenePerception will be automatic called when app is closed
WVR_Result startPerceptionResult_2d = WVR_StartScenePerception(WVR_ScenePerceptionTarget_2dPlane);
if (WVR_Success != startPerceptionResult_2d)
    LOGE("WVR_StartScenePerception init fail");

4-2. Get the Plane data - WVR_GetScenePlanes()

Application should call WVR_GetScenePlanes() twice in each frame to get the number of the planes and planes data.

// WVR API get scene num and allocate memory to get data
WVR_ScenePlane *mTempPlane;
uint32_t mTempPlaneCount;
WVR_GetScenePlanes(nullptr, 0, &mTempPlaneCount, WVR_PoseOriginModel_OriginOnHead, nullptr);
mTempPlane = (WVR_ScenePlane *)malloc(sizeof(WVR_ScenePlane) * mTempPlaneCount);
WVR_GetScenePlanes(nullptr, mTempPlaneCount, &mTempPlaneCount, WVR_PoseOriginModel_OriginOnHead, mTempPlane);
LOGI("get planes =%u", mTempPlaneCount);

// compare if new plane appear
if(planeChanged(mTempPlaneCount, mTempPlane)){
    shouldUpdatePlaneVertex=true;
}

// clear previous data
pm->clearPlanes();
if (mPlane != nullptr)
{
    free(mPlane);
    mPlane = nullptr;
}

// replace new data
mPlane = mTempPlane;
mPlaneCount=mTempPlaneCount;
for (int a = 0; a < mPlaneCount; a++)
{
    // print name
    LOGI("plane[%d] name=%s", a, mPlane[a].semanticName.name);
    pm->addPlane(a, convertToMatrix4(mPlane[a].pose), Vector3(mPlane[a].extent.width, mPlane[a].extent.height, 1));
}

void PlaneManager::addPlane(int index, Matrix4 pose, Vector3 scale)
{
    planesVector.push_back(Plane{pose, scale, noRay, index});
}

4-3. Render Plane object

Application can use the pose and extent to the render rectangles.

glUniform4f(mColor, 0.0f, 0.8f, 0.0f, 1.0f);
matrix.push_back(projection * eye * view * planesVector[a].mPose);
glUniformMatrix4fv(mMatrixLocation, 1, false, matrix[a].get());
glUniform3f(mScale, planesVector[a].mScale.x, planesVector[a].mScale.y, planesVector[a].mScale.z);//scale is define by plane extent
mVAO->bindVAO();
glDrawElements(GL_TRIANGLES, singleIndexArray.size(), GL_UNSIGNED_INT, 0);
mVAO->unbindVAO();
_images/renderPlane.png

4-4. Get plane mesh data - WVR_GetSceneMeshBuffer()

Plane mesh vertex and element index can be got via sending meshBufferId to WVR_GetSceneMeshBuffer.

WVR_SceneMeshBuffer buffer0 = {0, 0, nullptr, 0, 0, nullptr};

// get sceneMesh size
if (WVR_GetSceneMeshBuffer(mPlane[planes[b].scenePlaneIndex].meshBufferId, WVR_PoseOriginModel_OriginOnHead, &buffer0) != WVR_Success)
{
    LOGE("Fail to get plane sceneMesh size");
    sceneMesh2D->clearAllData();
}
else
    LOGI("Get sceneMesh index size=%d", buffer0.indexCountOutput);

std::vector<WVR_Vector3f> vertexBuffer(buffer0.vertexCountOutput);
std::vector<uint32_t> indexBuffer(buffer0.indexCountOutput);
WVR_SceneMeshBuffer buffer1 = {buffer0.vertexCountOutput, buffer0.vertexCountOutput, vertexBuffer.data(), buffer0.indexCountOutput, buffer0.indexCountOutput, indexBuffer.data()};

// get sceneMesh data
if (WVR_GetSceneMeshBuffer(mPlane[planes[b].scenePlaneIndex].meshBufferId, WVR_PoseOriginModel_OriginOnHead, &buffer1) != WVR_Success)
{
    LOGE("Fail to get sceneMesh data");
    sceneMesh2D->clearAllData();
}
sceneMesh2D->mMeshIndex.push_back(indexBuffer);
sceneMesh2D->mMeshVertex.push_back(vertexBuffer);

4-5. Render plane mesh data

Developer can simply render plane meshes by the vertex and the index with glDrawElement().

// push data to GPU (only when the vertice were changed)
mVAO->bindVAO();
mVAO->bindArrayBuffer();
int stride = 6 * sizeof(float);
GLuint offset = 0;
float *ptr = (float *)meshData->mMeshOrder[a].data();
glBufferData(GL_ARRAY_BUFFER, sizeof(float) * meshData->mMeshOrder[a].size(), ptr, GL_STATIC_DRAW);
glEnableVertexAttribArray(0); // populate vertex
glVertexAttribPointer(0, 3, GL_FLOAT, false, stride, (const void *)offset);
offset += sizeof(float) * 3;
glEnableVertexAttribArray(1); // populate color
glVertexAttribPointer(1, 3, GL_FLOAT, false, stride, (const void *)offset);
mVAO->unbindVAO();
mVAO->unbindArrayBuffer();

// render
meshData->mVertexSize.push_back(meshData->mMeshOrder[a].size() / 6);
LOGI("render %d vertex", meshData->mVertexSize[a]);
glUniformMatrix4fv(mMatrixLocation, 1, false, matrix.get());
mVAO->bindVAO();
glDrawArrays(GL_TRIANGLES, 0, meshData->mVertexSize[a]);
mVAO->unbindVAO();
_images/renderPlaneMesh.png

5. Scene Object & Object Mesh

5-1. Init

In order to use 3D object API, developer should call WVR_StartScene() and WVR_StartScenePerception() before calling any plane API.

// Loading the WVR scene
WVR_Result result = WVR_StartScene();
if (result != WVR_Success)
{
    LOGE("WVR_StartScene init fail %d", result);
    return false;
}

// start perception before get scene plane (need permission)
// can call WVR_StopScenePerception when you won't need to update data
// WVR_StopScenePerception will be automatic called when app is closed
WVR_Result startPerceptionResult_object = WVR_StartScenePerception(WVR_ScenePerceptionTarget_3dObject);
if (WVR_Success != startPerceptionResult_object)
    LOGE("WVR_StartScenePerception init fail");

5-2. Get the 3D Object data - WVR_GetSceneObjects()

Application should call WVR_GetSceneObjects() twice in each frame to get the number of the 3D object and 3D object data.

// WVR API get scene num and allocate memory to get data
WVR_SceneObject *mTempObject;
uint32_t mTempObjectCount;
WVR_GetSceneObjects(0, &mTempObjectCount, WVR_PoseOriginModel_OriginOnHead, nullptr);
mTempObject = (WVR_SceneObject *)malloc(sizeof(WVR_SceneObject) * mTempObjectCount);
WVR_GetSceneObjects(mTempObjectCount, &mTempObjectCount, WVR_PoseOriginModel_OriginOnHead, mTempObject);
LOGI("get objects =%u", mTempObjectCount);

if(objectChanged(mTempObjectCount, mTempObject)){
    shouldUpdateObjectVertex=true;
}

// clear previous data
om->clearSceneObjects();
if (mObject != nullptr)
{
    free(mObject);
    mObject = nullptr;
}

// copy to vector
mObject = mTempObject;
mObjectCount=mTempObjectCount;
for (int a = 0; a < mObjectCount; a++)
{
    // print name
    LOGI("object[%d] name=%s", a, mObject[a].semanticName.name);
    om->addSceneObject(a, convertToMatrix4(mObject[a].pose), Vector3(mObject[a].extent.width, mObject[a].extent.height, mObject[a].extent.depth));
}

void SceneObjectManager::addSceneObject(int index, Matrix4 pose, Vector3 scale)
{
    objectsVector.push_back(SceneObject{pose, scale, index});
}

5-3. Render 3D Object

Application can use the pose and extent to the render rectangles.

Matrix4 matrix = projection * eye * view * objectsVector[a].mPose;
glUniformMatrix4fv(mMatrixLocation, 1, false, matrix.get());
glUniform4f(mColor, 0.3f, 0.3f, 0.6f, 0.3f);
glUniform3f(mScale, objectsVector[a].mScale.x, objectsVector[a].mScale.y, objectsVector[a].mScale.z);
mVAO->bindVAO();
glDrawElements(GL_TRIANGLES, singleIndexArray.size(), GL_UNSIGNED_INT, 0);
mVAO->unbindVAO();
_images/renderObject.png

5-4. Get 3D Object mesh data - WVR_GetSceneMeshBuffer()

Object mesh vertex and element index can be got via sending meshBufferId to WVR_GetSceneMeshBuffer.

WVR_SceneMeshBuffer buffer0 = {0, 0, nullptr, 0, 0, nullptr};

// get sceneMesh size
if (WVR_GetSceneMeshBuffer(mObject[objects[b].index].meshBufferId, WVR_PoseOriginModel_OriginOnHead, &buffer0) != WVR_Success)
{
    LOGE("Fail to get object mesh size. Clear all");
    sceneMesh3D->clearAllData();
}
else
    LOGI("Get mesh index size=%d", buffer0.indexCountOutput);

std::vector<WVR_Vector3f> vertexBuffer(buffer0.vertexCountOutput);
std::vector<uint32_t> indexBuffer(buffer0.indexCountOutput);
WVR_SceneMeshBuffer buffer1 = {buffer0.vertexCountOutput, buffer0.vertexCountOutput, vertexBuffer.data(),
                                buffer0.indexCountOutput, buffer0.indexCountOutput, indexBuffer.data()};

// get sceneMesh data
if (WVR_GetSceneMeshBuffer(mObject[objects[b].index].meshBufferId, WVR_PoseOriginModel_OriginOnHead, &buffer1) != WVR_Success)
{
    LOGE("Fail to get object mesh data. Clear all");
    sceneMesh3D->clearAllData();
    return;
}
sceneMesh3D->mMeshIndex.push_back(indexBuffer);
sceneMesh3D->mMeshVertex.push_back(vertexBuffer);

5-5. Render 3D Object mesh data

Developer can simply render 3D object meshes by the vertex and the index with glDrawElement().

// push data to GPU (only when the vertice were changed)
mVAO->bindVAO();
mVAO->bindArrayBuffer();
int stride = 6 * sizeof(float);
GLuint offset = 0;
float *ptr = (float *)meshData->mMeshOrder[a].data();
glBufferData(GL_ARRAY_BUFFER, sizeof(float) * meshData->mMeshOrder[a].size(), ptr, GL_STATIC_DRAW);
glEnableVertexAttribArray(0); // populate vertex
glVertexAttribPointer(0, 3, GL_FLOAT, false, stride, (const void *)offset);
offset += sizeof(float) * 3;
glEnableVertexAttribArray(1); // populate color
glVertexAttribPointer(1, 3, GL_FLOAT, false, stride, (const void *)offset);
mVAO->unbindVAO();
mVAO->unbindArrayBuffer();

// render
meshData->mVertexSize.push_back(meshData->mMeshOrder[a].size() / 6);
LOGI("render %d vertex", meshData->mVertexSize[a]);
glUniformMatrix4fv(mMatrixLocation, 1, false, matrix.get());
mVAO->bindVAO();
glDrawArrays(GL_TRIANGLES, 0, meshData->mVertexSize[a]);
mVAO->unbindVAO();
_images/renderObjectMesh.png

6. Collider Mesh & Visual Mesh

6-1. Init

In order to use mesh API, developer should call WVR_StartScene() and WVR_StartScenePerception() before calling any mesh APIs.

// Loading the WVR scene
WVR_Result result = WVR_StartScene();
if (result != WVR_Success)
{
    LOGE("WVR_StartScene init fail %d", result);
    return false;
}

// start perception before get scene mesh (need permission)
// can call WVR_StopScenePerception when you won't need to update data
// WVR_StopScenePerception will be automatic called when app is closed
WVR_Result startPerceptionResult_sceneMesh = WVR_StartScenePerception(WVR_ScenePerceptionTarget_SceneMesh);
if (WVR_Success != startPerceptionResult_sceneMesh)
    LOGE("WVR_StartScenePerception init fail");

6-2. Check the scene perception state - WVR_GetScenePerceptionState()

Application should check if HMD device can recognize the corresponding mesh. If the state is not equal to WVR_ScenePerceptionState_Complete, application should not use WVR_GetSceneMeshBuffer() to get the mesh data.

WVR_GetScenePerceptionState(WVR_ScenePerceptionTarget_SceneMesh,&spstate);
if(spstate==WVR_ScenePerceptionState_Completed){
    // get mesh data
    ...
}
else
{
    LOGI("Scene perception not complete");
    mSceneMeshBufferVisual_index.clear();
    mSceneMeshBufferVisual_vertex.clear();
}

6-3. Get the existing collider/visual meshBufferId - WVR_GetSceneMeshes()

Application should call WVR_GetSceneMeshes() to get the newest existing meshBufferId before calling WVR_GetSceneMeshBuffer(). If any scene mesh vertex is changed, all meshBufferId will be changed as well.

//ColliderMesh
uint32_t  count_mesh;
//get number of sceneMeshes
if(WVR_GetSceneMeshes(WVR_SceneMeshType_ColliderMesh, 0, &count_mesh, nullptr)!=WVR_Success)
    LOGE("Fail to get sceneMeshes count");
else
    LOGI("SceneMesh ID size=%d", count_mesh);

//Get sceneMeshes ID
std::vector<WVR_SceneMesh> meshes(count_mesh);
if(WVR_GetSceneMeshes(WVR_SceneMeshType_ColliderMesh, count_mesh, &count_mesh, meshes.data())!=WVR_Success)
    LOGE("Fail to get sceneMeshes ID");

//visualMesh
if(WVR_GetSceneMeshes(WVR_SceneMeshType_VisualMesh, 0, &count_mesh, nullptr)!=WVR_Success)
/*same as collider mesh...*/

6-4. Get the collider/visual mesh data - WVR_GetSceneMeshBuffer ()

Application could use meshBufferId and WVR_GetSceneMeshBuffer() to get the mesh vertex and element index. Owing to performance concerns, we strongly recommend only using WVR_GetSceneMeshBuffer() when the scene mesh vertex is changed.

//only update data when any vertex had changed
//If any mesh data had changed, all buffer id get from WVR_GetSceneMeshes() would be change
if(lastVisualMeshID!=meshes[0].meshBufferId){
    LOGI("visual sceneMesh data should be updated");
    WVR_SceneMeshBuffer buffer0={0, 0, nullptr, 0, 0, nullptr};
    //get sceneMesh size
    if(WVR_GetSceneMeshBuffer(meshes[a].meshBufferId, WVR_PoseOriginModel_OriginOnHead, &buffer0)!=WVR_Success){
        LOGE("Fail to get visual sceneMesh size");
        mSceneMeshBufferVisual_index.clear();
        mSceneMeshBufferVisual_vertex.clear();
        lastVisualMeshID=-1;
        return;
    }
    else
        LOGI("Get sceneMesh index size=%d", buffer0.indexCountOutput);

    std::vector<WVR_Vector3f> vertexBuffer(buffer0.vertexCountOutput);
    std::vector<uint32_t> indexBuffer(buffer0.indexCountOutput);
    WVR_SceneMeshBuffer buffer1 = {buffer0.vertexCountOutput, buffer0.vertexCountOutput,vertexBuffer.data(), buffer0.indexCountOutput, buffer0.indexCountOutput, indexBuffer.data()};

    //get sceneMesh data
    if(WVR_GetSceneMeshBuffer(meshes[a].meshBufferId, WVR_PoseOriginModel_OriginOnHead, &buffer1)!=WVR_Success){
        LOGE("Fail to get sceneMesh data");
        mSceneMeshBufferVisual_index.clear();
        mSceneMeshBufferVisual_vertex.clear();
        lastVisualMeshID=-1;
        return;
    }
    mSceneMeshBufferVisual_index.push_back(indexBuffer);
    mSceneMeshBufferVisual_vertex.push_back(vertexBuffer);
}

6-5. Render collider/visual mesh

Developer can render 3D mesh simply by vertex and index with glDrawElement(). In addition, developer can also modify the vertex and index to create a better visual effect. Owing to the large vertex and element data, we strongly recommend not to push the vertex to GPU in each frame. Please push the vertex to GPU when the the vertex data is changed.

//Modify color
Vector3 selectColor(int num){
    Vector3 color;
    switch (num) {
    case 1:
        color.x = 238.0 / 255;
        color.y = 132.0 / 255;
        color.z = 117.0 / 255;
        break;
    /*skip ...*/
    case 10:
        color.x = 157.0 / 255;
        color.y = 28.0 / 255;
        color.z = 200.0 / 255;
        break;
    }
    return color;
}
//modify vertex and element for better visual effect
void getOrderMesh(){
    int count;
    Vector3 color;
    mMeshOrder.clear();
    std::vector<float> buffer;
    for (int a = 0; a < mMeshIndex.size(); a++) {
        count = 0;
        buffer.clear();
        for (int b = 0; b < mMeshIndex[a].size(); b = b + 3){
            count = count % 10 + 1;
            color = selectColor(count);
            buffer.push_back(mMeshVertex[a][mMeshIndex[a][b]].v[0]);
            buffer.push_back(mMeshVertex[a][mMeshIndex[a][b]].v[1]);
            buffer.push_back(mMeshVertex[a][mMeshIndex[a][b]].v[2]);
            buffer.push_back(color.x);
            buffer.push_back(color.y);
            buffer.push_back(color.z);
        }
        mMeshOrder.push_back(buffer);
    }
}

// push data to GPU (only when the vertice were changed)
mVAO->bindVAO();
mVAO->bindArrayBuffer();
int stride = 6 * sizeof(float);
GLuint offset = 0;
float *ptr = (float *)meshData->mMeshOrder[a].data();
glBufferData(GL_ARRAY_BUFFER, sizeof(float) * meshData->mMeshOrder[a].size(), ptr, GL_STATIC_DRAW);
glEnableVertexAttribArray(0); // populate vertex
glVertexAttribPointer(0, 3, GL_FLOAT, false, stride, (const void *)offset);
offset += sizeof(float) * 3;
glEnableVertexAttribArray(1); // populate color
glVertexAttribPointer(1, 3, GL_FLOAT, false, stride, (const void *)offset);
mVAO->unbindVAO();
mVAO->unbindArrayBuffer();

// render
meshData->mVertexSize.push_back(meshData->mMeshOrder[a].size() / 6);
LOGI("render %d vertex", meshData->mVertexSize[a]);
glUniformMatrix4fv(mMatrixLocation, 1, false, matrix.get());
mVAO->bindVAO();
glDrawArrays(GL_TRIANGLES, 0, meshData->mVertexSize[a]);
mVAO->unbindVAO();
  • Render Visual Mesh
_images/visualMesh.png
  • Render Collider Mesh
_images/colliderMesh.png

7. Others

While developer use the last section features, developer should align WVR_PoseOriginModel in all WAVE API. If nothing is incorrect, developer can obtain the MR content with the correct position as below.

_images/renderAll.png