$title 'Test Connect agent RawCSVReader' (CARCR,SEQ=934)

$onText
This test ensures the correctness of the Connect agent RawCSVReader.

Contributor: Michael Bussieck, February 2023
$offText

$log --- Using Python library %sysEnv.GMSPYTHONLIB%

$onEchoV > carcr.csv
TRUE,1
FALSE,2
1/1/2008,3
inf,4
-inf,5
na,6
nan,7
eps,8
Eps,9
undef,10
39448,11
short,
a23456789012345678901234567890123456789012345678901234567890123,
a234567890123456789012345678901234567890123456789012345678901234,
a234567890123456789012345678901234567890123456789012345678901~1,
a23456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678,
'test',
"test",
t"es"t,
t'es't,
t'e"s't,
This is some label that can not be represented in GAMS since it is too long
This is some label that can not be represented in GAMS since it is ALSO too long
This is ANOTHER label that can not be represented in GAMS since it is too long
This is A THIRD label that can not be represented in GAMS since it is too long
$offEcho

$onEchoV > t.gms
Set c(*) Columns / %C%1*%C%2 /;
Set r(*) Rows / %R%1*%R%25 /;

$onUNDF
Parameter vf(*,*) Cells with numerical value /
%R%1 .%C%1 1,
%R%1 .%C%2 1,
%R%2 .%C%2 2,
%R%3 .%C%2 3,
%R%4 .%C%1 +Inf,
%R%4 .%C%2 4,
%R%5 .%C%1 -Inf,
%R%5 .%C%2 5,
%R%6 .%C%1 NA,
%R%6 .%C%2 6,
%R%7 .%C%1 Undf,
%R%7 .%C%2 7,
%R%8 .%C%1 Eps,
%R%8 .%C%2 8,
%R%9 .%C%1 Eps,
%R%9 .%C%2 9,
%R%10.%C%1 Undf,
%R%10.%C%2 10,
%R%11.%C%1 39448,
%R%11.%C%2 11 /;
$offUNDF

Set vs(*,*) Cells with explanatory text /
%R%1. %C%1 'TRUE',
%R%2. %C%1 'FALSE',
%R%3. %C%1 '1/1/2008',
%R%4. %C%1 inf,
%R%5. %C%1 -inf,
%R%6. %C%1 na,
%R%7. %C%1 nan,
%R%8. %C%1 eps,
%R%9. %C%1 Eps,
%R%10.%C%1 undef,
%R%12.%C%1 short,
%R%13.%C%1 a23456789012345678901234567890123456789012345678901234567890123,
%R%14.%C%1 a234567890123456789012345678901234567890123456789012345678901234,
%R%15.%C%1 a234567890123456789012345678901234567890123456789012345678901~1,
%R%16.%C%1 a2345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
%R%17.%C%1 "'test'"
%R%18.%C%1 test
%R%19.%C%1 't"es"t'
%R%20.%C%1 "t'es't"
%R%21.%C%1 "t'e's't"
%R%22.%C%1 "This is some label that can not be represented in GAMS since it is too long"
%R%23.%C%1 "This is some label that can not be represented in GAMS since it is ALSO too long"
%R%24.%C%1 "This is ANOTHER label that can not be represented in GAMS since it is too long"
%R%25.%C%1 "This is A THIRD label that can not be represented in GAMS since it is too long"
/;

Set %vu%(*,*,*) Cells with potential GAMS label /
%R%1.%C%1.'TRUE' TRUE,
%R%2.%C%1.'FALSE' FALSE,
%R%3.%C%1.'1/1/2008' '1/1/2008',
%R%4.%C%1.'inf' inf,
%R%5.%C%1.'-inf' -inf,
%R%6.%C%1.'na' na,
$ifThen %READASSTRING%==False
%R%1.%C%2.'1.0' 1.0,
%R%2.%C%2.'2.0' 2.0,
%R%3.%C%2.'3.0' 3.0,
%R%4.%C%2.'4.0' 4.0,
%R%5.%C%2.'5.0' 5.0,
%R%6.%C%2.'6.0' 6.0,
%R%7.%C%2.'7.0' 7.0,
%R%8.%C%2.'8.0' 8.0,
%R%9.%C%2.'9.0' 9.0,
%R%10.%C%2.'10.0' 10.0,
%R%11.%C%2.'11.0' 11.0,
$else
%R%1.%C%2.'1' 1,
%R%2.%C%2.'2' 2,
%R%3.%C%2.'3' 3,
%R%4.%C%2.'4' 4,
%R%5.%C%2.'5' 5,
%R%6.%C%2.'6' 6,
%R%7.%C%2.'7' 7,
%R%8.%C%2.'8' 8,
%R%9.%C%2.'9' 9,
%R%10.%C%2.'10' 10,
%R%11.%C%2.'11' 11,
$endIf
%R%7.%C%1.'nan' nan,
%R%8.%C%1.'eps' eps,
%R%9.%C%1.'eps' Eps,
%R%10.%C%1.'undef' undef,
%R%11.%C%1.'39448' 39448,
%R%12.%C%1.'short' short,
%R%13.%C%1.'a23456789012345678901234567890123456789012345678901234567890123' a23456789012345678901234567890123456789012345678901234567890123,
%R%14.%C%1.'a234567890123456789012345678901234567890123456789012345678901~1' a234567890123456789012345678901234567890123456789012345678901234,
%R%15.%C%1.'a234567890123456789012345678901234567890123456789012345678901~2' a234567890123456789012345678901234567890123456789012345678901~1,
%R%16.%C%1.'a234567890123456789012345678901234567890123456789012345678901~3' a2345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234
%R%17.%C%1."'test'" "'test'"
%R%18.%C%1.test test
%R%19.%C%1.'t"es"t' 't"es"t'
%R%20.%C%1."t'es't" "t'es't"
%R%22.%C%1.'This is some label that can not be represented in GAMS since ~1' "This is some label that can not be represented in GAMS since it is too long"
%R%23.%C%1.'This is some label that can not be represented in GAMS since ~2' "This is some label that can not be represented in GAMS since it is ALSO too long"
%R%24.%C%1.'This is ANOTHER label that can not be represented in GAMS sin~1' "This is ANOTHER label that can not be represented in GAMS since it is too long"
%R%25.%C%1.'This is A THIRD label that can not be represented in GAMS sin~1' "This is A THIRD label that can not be represented in GAMS since it is too long"
/;
$offEcho

*** test expected errors ***
$log Test raising an exception for existing symbol name
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      connect.container.addSet("r")
- RawCSVReader:
    file: carcr.csv
    readAsString: False
    rowLabel: rxx
    columnLabel: cxx
    vuName: vux
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

*********************************************************

$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx ref.gdx --READASSTRING=True --vu=vu --S=S --R=R --C=C
$onEmbeddedCode Connect:
- RawCSVReader:
    file: carcr.csv
- GDXWriter:
    file: ctrcrout.gdx
$offEmbeddedCode

$call.checkErrorLevel gdxdiff ref.gdx ctrcrout.gdx > %system.NullFile%

$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx ref.gdx --READASSTRING=False --vu=vu --S=S --R=R --C=C
$onEmbeddedCode Connect:
- RawCSVReader:
    file: carcr.csv
    readAsString: False
- GDXWriter:
    file: ctrcrout.gdx
$offEmbeddedCode

$call.checkErrorLevel gdxdiff ref.gdx ctrcrout.gdx > %system.NullFile%

$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx ref.gdx --READASSTRING=True --vu=vux --R=rxx --C=cxx
$onEmbeddedCode Connect:
- RawCSVReader:
    file: carcr.csv
    rowLabel: rxx
    columnLabel: cxx
    vuName: vux
- GDXWriter:
    file: ctrcrout.gdx
    symbols: all
$offEmbeddedCode

$call.checkErrorLevel gdxdiff ref.gdx ctrcrout.gdx > %system.NullFile%

$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx ref.gdx --READASSTRING=False --vu=vux --R=rxx --C=cxx
$onEmbeddedCode Connect:
- RawCSVReader:
    file: carcr.csv
    readAsString: False
    rowLabel: rxx
    columnLabel: cxx
    vuName: vux
- GDXWriter:
    file: ctrcrout.gdx
$offEmbeddedCode

$call.checkErrorLevel gdxdiff ref.gdx ctrcrout.gdx > %system.NullFile%

$log Test that symbols have empty DataFrames (Not None records) when they have no content
$onechoV > carcr_empty_df.csv
a,b,c,
d,e,f,
g,h,i,
$offEcho

$onEmbeddedCode Connect:
- RawCSVReader:
    file: carcr_empty_df.csv
    rowLabel: rxx
    columnLabel: cxx
- PythonCode:
    code: |
      data = connect.container['vf'].records
      expected = ['r', 'c', 'value']
      if data is None or not data.empty:
        raise Exception(f"Symbol >vf< is expected to have an empty DataFrame!")

      if (data.columns != expected).any() :
        raise Exception("Unexpected columns for DataFrame")
$offEmbeddedCode