$title 'Test gams.core.numpy API raw mode' (GAMS2NUMPY02,SEQ=827)

$onText
Test gams.core.numpy Python API raw mode using the GAMS Embedded Code facility

Contributor: Clemens Westphal, July 2020
$offText

$log --- Using Python library %sysEnv.GMSPYTHONLIB%

set i /i1*i10/;
set j /j1*j10/;
set k /1*10/;
set l /1*10/;

parameter p(i,j), p2(k,l);
p(i,j) = uniform(0,10);
p2(k,l) = uniform(0,10);
p2('1','1') = +inf;

execute_unload 'p_reference.gdx', p;
execute_unload 'ijp_reference.gdx', i j p;
execute_unload 'klp2_reference.gdx', k l p2;

*Test raw mode
embeddedCode Python:
    import numpy as np
    import gams.core.numpy as gnp
    from gams.core.gdx import *
    from gams.core.gmd import *

    gams2np = gnp.Gams2Numpy(r"%gams.sysdir%.")
    gdx = new_gdxHandle_tp()
    gmd1 = new_gmdHandle_tp()
    gmd2 = new_gmdHandle_tp()

    rc, msg = gmdCreate(gmd1, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)

    rc, msg = gmdCreate(gmd2, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)

    rc, msg = gdxCreate(gdx, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)

    if not gdxOpenRead(gdx, 'ijp_reference.gdx')[0]:
        raise Exception("Error opening GDX file 'ijp_reference.gdx'")
    i_keys, i_values = gams2np.gdxReadSymbolRaw(gdx, 'i')
    j_keys, j_values = gams2np.gdxReadSymbolRaw(gdx, 'j')
    p_keys, p_values = gams2np.gdxReadSymbolRaw(gdx, 'p')
    uels = list(gams.get('*'))
    gdxClose(gdx)
    if not gdxFree(gdx):
        raise Exception("Error in gdxFree")

    rc, msg = gdxCreate(gdx, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    if not gdxOpenWrite(gdx, 'p_gdx1.gdx', '')[0]:
        raise Exception("Error creating GDX file 'p_gdx1.gdx'")
    gdxUELRegisterStrStart(gdx)
    for l in uels:
        gdxUELRegisterStr(gdx, l)
    gdxUELRegisterDone(gdx)
    gams2np.gdxWriteSymbolRaw(gdx, 'p', '', 2, GMS_DT_PAR, 0, p_keys, p_values)
    if gdxClose(gdx):
        raise Exception("Error closing GDX file")
    if not gdxFree(gdx):
        raise Exception("Error in gdxFree")

    rc, msg = gdxCreate(gdx, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    if not gdxOpenWrite(gdx, 'ijp_gdx.gdx', '')[0]:
        raise Exception("Error creating GDX file 'ijp_gdx.gdx'")
    gdxUELRegisterStrStart(gdx)
    for l in uels:
        gdxUELRegisterStr(gdx, l)
    gdxUELRegisterDone(gdx)
    gams2np.gdxWriteSymbolRaw(gdx, 'i', '', 1, GMS_DT_SET, 0, i_keys, i_values)
    gams2np.gdxWriteSymbolRaw(gdx, 'j', '', 1, GMS_DT_SET, 0, j_keys, j_values)
    gams2np.gdxWriteSymbolRaw(gdx, 'p', '', 2, GMS_DT_PAR, 0, p_keys, p_values, ['i', 'j'])
    if gdxClose(gdx):
        raise Exception("Error closing GDX file")
    if not gdxFree(gdx):
        raise Exception("Error in gdxFree")

    rc = new_intp()
    for l in uels:
        gmdMergeUel(gmd1, l)
    p_symPtr = gmdAddSymbolPy(gmd1, 'p', 2, GMS_DT_PAR, 0, '', rc)
    if not intp_value(rc):
        raise Exception("Error adding symbol to GMD")

    p_values2 = p_values.copy()
    p_values2[:,-1] = 3.14 # add wrong values
    gams2np.gmdFillSymbolRaw(gmd1, p_symPtr, p_keys, p_values2)

    # add a duplicate record to the array and write correct values using merge=True
    p_keys = np.insert(p_keys, 0, np.array([1, 11]), axis=0)
    p_values = np.insert(p_values, 0, np.array([1.23]), axis=0)

    gams2np.gmdFillSymbolRaw(gmd1, p_symPtr, p_keys, p_values, merge=True)
    gmdWriteGDX(gmd1, 'p_gmd1.gdx', False)
    rc = new_intp()
    p_keys, p_values = gams2np.gmdReadSymbolRaw(gmd1, 'p')
    uels = gams2np.gmdGetUelList(gmd1)
    for l in uels[1:]:
        gmdMergeUel(gmd2, l)
    p_symPtr = gmdAddSymbolPy(gmd2, 'p', 2, GMS_DT_PAR, 0, '', rc)
    if not intp_value(rc):
        raise Exception("Error adding symbol to GMD")
    gams2np.gmdFillSymbolRaw(gmd2, p_symPtr, p_keys, p_values)
    gmdWriteGDX(gmd2, 'p_gmd2.gdx', False)

    if not gmdFree(gmd1):
        raise Exception("Error in gmdFree")
    if not gmdFree(gmd2):
        raise Exception("Error in gmdFree")

    # Test parameter relaxedType=True (GMD)
    gmd3 = new_gmdHandle_tp()
    rc, msg = gmdCreate(gmd3, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    rc = new_intp()
    uels = list(gams.get('*'))
    for l in uels:
        gmdMergeUel(gmd3, l)
    k_symPtr = gmdAddSymbolPy(gmd3, 'k', 1, GMS_DT_SET, 0, '', rc)
    if not intp_value(rc):
        raise Exception("Error adding symbol to GMD")
    k_keys = np.array([[str(uel)] for uel in gams.get('k', keyType=KeyType.INT)])
    k_values = None

    gams2np.gmdFillSymbolRaw(gmd3, k_symPtr, k_keys, k_values, relaxedType=True)

    l_symPtr = gmdAddSymbolPy(gmd3, 'l', 1, GMS_DT_SET, 0, '', rc)
    if not intp_value(rc):
        raise Exception("Error adding symbol to GMD")
    gams2np.gmdFillSymbolRaw(gmd3, l_symPtr, k_keys, k_values, relaxedType=True)

    p2_symPtr = gmdAddSymbolPy(gmd3, 'p2', 2, GMS_DT_PAR, 0, '', rc)
    if not intp_value(rc):
        raise Exception("Error adding symbol to GMD")
    p2_keys = np.array(gams.get("p2", keyFormat=KeyFormat.FLAT, keyType=KeyType.INT, valueFormat=ValueFormat.SKIP), dtype=str)
    p2_values = np.array(gams.get("p2", keyFormat=KeyFormat.SKIP, valueFormat=ValueFormat.TUPLE), dtype=object)
    p2_values[0] = 'inf'

    gams2np.gmdFillSymbolRaw(gmd3, p2_symPtr, p2_keys, p2_values, relaxedType=True)
    gmdWriteGDX(gmd3, 'klp2_gmd.gdx', False)
    if not gmdFree(gmd3):
        raise Exception("Error in gmdFree")

    # Test parameter relaxedType=True (GDX)
    rc, msg = gdxCreate(gdx, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    if not gdxOpenWrite(gdx, 'klp2_gdx.gdx', '')[0]:
        raise Exception("Error creating GDX file 'klp2_gdx.gdx'")
    gdxUELRegisterStrStart(gdx)
    for l in uels:
        gdxUELRegisterStr(gdx, l)
    gdxUELRegisterDone(gdx)
    gams2np.gdxWriteSymbolRaw(gdx, 'k', '', 1, GMS_DT_SET, 0, k_keys, k_values, relaxedType=True)
    gams2np.gdxWriteSymbolRaw(gdx, 'l', '', 1, GMS_DT_SET, 0, k_keys, k_values, relaxedType=True)
    gams2np.gdxWriteSymbolRaw(gdx, 'p2', '',2, GMS_DT_PAR, 0, p2_keys, p2_values, relaxedType=True)
    if gdxClose(gdx):
        raise Exception("Error closing GDX file")
    if not gdxFree(gdx):
        raise Exception("Error in gdxFree")

    # Test map mode (GDX)
    rc, msg = gdxCreate(gdx, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    if not gdxOpenWrite(gdx, 'ijp_gdx_map.gdx', '')[0]:
        raise Exception("Error creating GDX file 'ijp_gdx_map.gdx'")
    gdxUELRegisterStrStart(gdx)
    for l in uels:
        gdxUELRegisterStr(gdx, l)
    gdxUELRegisterDone(gdx)

    np.random.shuffle(i_keys)
    np.random.shuffle(j_keys)
    p = np.random.permutation(len(p_keys))
    p_keys = p_keys[p]
    p_values = p_values[p]

    gams2np.gdxWriteSymbolMap(gdx, 'i', '', 1, GMS_DT_SET, 0, i_keys, None)
    gams2np.gdxWriteSymbolMap(gdx, 'j', '', 1, GMS_DT_SET, 0, j_keys, None)
    gams2np.gdxWriteSymbolMap(gdx, 'p', '', 2, GMS_DT_PAR, 0, p_keys, p_values)
    if gdxClose(gdx):
        raise Exception("Error closing GDX file")
    if not gdxFree(gdx):
        raise Exception("Error in gdxFree")

    # Test cat mode (GDX and GMD comparison)
    rc, msg = gdxCreate(gdx, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    if not gdxOpenRead(gdx, 'ijp_reference.gdx')[0]:
        raise Exception("Error opening GDX file 'ijp_reference.gdx'")
    i_keys_cat_gdx, i_values_cat_gdx, i_labels_cat_gdx = gams2np.gdxReadSymbolCat(gdx, 'i')
    j_keys_cat_gdx, j_values_cat_gdx, j_labels_cat_gdx = gams2np.gdxReadSymbolCat(gdx, 'j')
    p_keys_cat_gdx, p_values_cat_gdx, p_labels_cat_gdx = gams2np.gdxReadSymbolCat(gdx, 'p')
    if gdxClose(gdx):
        raise Exception("Error closing GDX file")
    if not gdxFree(gdx):
        raise Exception("Error in gdxFree")

    gmd4 = new_gmdHandle_tp()
    rc, msg = gmdCreate(gmd4, GMS_SSSIZE)
    if not rc:
        raise Exception(msg)
    rc = gmdInitFromGDX(gmd4, 'ijp_reference.gdx')
    if not rc:
        raise Exception("Error gmdInitFromGDX")
    i_keys_cat_gmd, i_values_cat_gmd, i_labels_cat_gmd = gams2np.gmdReadSymbolCat(gmd4, 'i')
    j_keys_cat_gmd, j_values_cat_gmd, j_labels_cat_gmd = gams2np.gmdReadSymbolCat(gmd4, 'j')
    p_keys_cat_gmd, p_values_cat_gmd, p_labels_cat_gmd = gams2np.gmdReadSymbolCat(gmd4, 'p')
    if not gmdFree(gmd4):
        raise Exception("Error in gmdFree")

    if (
        not np.array_equal(i_keys_cat_gdx, i_keys_cat_gmd)
        or not np.array_equal(i_values_cat_gdx, i_values_cat_gmd)
        or not np.array_equal(i_labels_cat_gdx, i_labels_cat_gmd)
    ):
        raise Exception("Reading cat data structures for 'i' not equal")

    if (
        not np.array_equal(j_keys_cat_gdx, j_keys_cat_gmd)
        or not np.array_equal(j_values_cat_gdx, j_values_cat_gmd)
        or not np.array_equal(j_labels_cat_gdx, j_labels_cat_gmd)
    ):
        raise Exception("Reading cat data structures for 'j' not equal")

    if (
        not np.array_equal(p_keys_cat_gdx, p_keys_cat_gmd)
        or not np.array_equal(p_values_cat_gdx, p_values_cat_gmd)
        or not np.array_equal(p_labels_cat_gdx, p_labels_cat_gmd)
    ):
        raise Exception("Reading cat data structures for 'p' not equal")
endEmbeddedCode

execute 'gdxdiff p_reference.gdx p_gdx1.gdx > %system.nullfile%';
abort$errorlevel 'p_reference.gdx and p_gdx1.gdx are different'

execute 'gdxdiff ijp_reference.gdx ijp_gdx.gdx > %system.nullfile%';
abort$errorlevel 'ijp_reference.gdx and ijp_gdx.gdx are different'

execute 'gdxdiff p_reference.gdx p_gmd1.gdx > %system.nullfile%';
abort$errorlevel 'p_reference.gdx and p_gmd1.gdx are different'

execute 'gdxdiff p_reference.gdx p_gmd2.gdx > %system.nullfile%';
abort$errorlevel 'p_reference.gdx and p_gmd2.gdx are different'

execute 'gdxdiff klp2_reference.gdx klp2_gmd.gdx > %system.nullfile%';
abort$errorlevel 'klp2_reference.gdx and klp2_gmd.gdx are different';

execute 'gdxdiff klp2_reference.gdx klp2_gdx.gdx > %system.nullfile%';
abort$errorlevel 'klp2_reference.gdx and klp2_gdx.gdx are different';

execute 'gdxdiff ijp_reference.gdx ijp_gdx_map.gdx > %system.nullfile%';
abort$errorlevel 'ijp_reference.gdx and ijp_gdx_map.gdx are different';