Striim 3.9.6 documentation

BinaryParser

Can be configured to parse a wide variety of binary formats. See Supported reader-parser combinations for compatible readers.

property

type

default value

notes

endian

java.lang.Boolean

False

False means the format is little endian, set to True if the format is big endian

metadata

java.lang.string

path from root or relative to the Striim program directory and the name of JSON file defining how to parse the data

StringTerminatedByNull

java.lang.Boolean

True

With the default value of True, strings must be terminated by a single null character (ASCII 00). If set to False, string length must be defined in the JSON file specified by the metadata property.

The output type of a source using BinaryParser is WAEvent.

The data types supported by the metadata JSON are:

type

length in bytes

BYTE

1

DOUBLE

8

FLOAT

4

INTEGER

4

LONG

8

SHORT

2

STRING

as set by stringColumnLength

You can find insorders.cmetadata.json, and bin.tql under the Striim program directory at docs/BinaryParser. To run this sample application:

1. Run insorders.c and copy its test.bin output to the Samples directory.

/***** insorders.c ******************************************
**                                                          *
** Generate order records in binary format                  *
** Usage insorders <recordcount>                            *
** Max recordcount 1000000                                  *
** Note                                                     *
**                                                          *
************************************************************/
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
/* Define some useful constants */
#define HIGH  1
#define MEDIUM 2
#define LOW 3
#define PRICE_TYPES 3
#define FILENAME2               "/Users/mahadevan/binary/test.bin"
#define MAX_LEN_CNAME           20
#define MAX_DIG_CNAME           10
#define NAMELEN                 20
#define MAX_ITEM_COUNT          200
#define EXPENSIVE_ITEM_PRICE    745.99
#define MEDIUM_ITEM_PRICE       215.99
#define LOW_ITEM_PRICE          34.79
 
/* Function prototypes */
int testdata(int numRecords);
double setPrice(int cid);
int main(int argc, char *argv[])
{
  int count = 1000;
  int ret = 1;
  /* just basic error checking for now is sufficient */
  
  
   printf(" Inserting %d Records\n", count);
  
  
  if ((ret = testdata(count)) !=0)
     printf("TestData Failed\n");
} /* End Main */
/* Start Function for test data generation */
int testdata(int numRecords)
{
  int i,nm1,nm2 = 0;
  
  /* Declare variables for random record creation \n")*/  
  int order_id=10000;
  int district_id=0;
  int warehouse_id = 100;
  char cname[numRecords][MAX_LEN_CNAME];  
  char PrefixName[NAMELEN]="John";
  char numRecStr[MAX_DIG_CNAME];
  int count_id=0;
  double price=0;
  /* time_t now; Just generate sysdate for now */
  bzero(numRecStr,MAX_DIG_CNAME);
  FILE *fptr_bin =fopen(FILENAME2, "w");
  if (fptr_bin==NULL) return -1;
   for (nm1=0; nm1 < numRecords; nm1++)
   {
      sprintf(numRecStr, " %d",nm1); 
      strcat(PrefixName,numRecStr); 
      strcpy(cname[nm1],PrefixName); 
      //printf("Generated Name is %s\n", cname[nm1]);
      /* Re-init to base root for name */
      strcpy(PrefixName, "John");
      /* Generate a random count of items between 0 and 20 */
      count_id = rand()%MAX_ITEM_COUNT;
      price = setPrice(count_id);
      
      printf("Price is %f\n",price);
      short cnamelen = strlen(cname[nm1]);
      /* Generate record with the following fields */
      fwrite((const void *) (&order_id), sizeof(int), 1, fptr_bin); order_id++;
      fwrite((const void *) (&district_id), sizeof(int), 1, fptr_bin); district_id++;
      fwrite((const void *) (&warehouse_id), sizeof(int), 1, fptr_bin); warehouse_id++;
      fwrite((const void *) (&cnamelen), sizeof(short), 1, fptr_bin); 
      fwrite((const void *) (cname[nm1]), sizeof(char), strlen(cname[nm1]), fptr_bin); 
      fwrite((const void *) (&count_id), sizeof(int), 1, fptr_bin); count_id++;
      fwrite((const void *) (&price), sizeof(double), 1, fptr_bin); 
  }
  fclose(fptr_bin);
  return 0;
}
/* Start setPrice */
  double setPrice(int cid)
  {
    short i;
    double price, total_price=0;
    
    i = rand()%PRICE_TYPES;
    switch (i) 
    {
      case(HIGH):
          price = EXPENSIVE_ITEM_PRICE;
      case(MEDIUM):
          price = MEDIUM_ITEM_PRICE;
      case(LOW):
          price = LOW_ITEM_PRICE;
     }
     total_price = cid*price;
     return total_price;
  } /* End setPrice */

2. Copy metadata.json to the Samples directory.

{
     "namespace": "test",
     "name": "SensorFeed",
     "version": "1.0",
     "type": "record",
     "fields": [
       {
        "name": "order_id",
        "type": "INTEGER",
         "position":"0"
        },        
       {
        "name": "district_id",
        "type": "INTEGER",
         "position":"1"
        },        
       {
        "name": "warehouse_id",
        "type": "INTEGER",
         "position":"2"
        },        
       {
        "name": "customer_name",
        "type": "STRING",
         "position":"3"
        },        
       {
        "name": "count_id",
        "type": "INTEGER",
         "position":"4"
        },        
       {
        "name": "price",
        "type": "DOUBLE",
         "position":"5"
        }        
      ]
    }

4. Load bin.tql and run the bin application.

CREATE APPLICATION bin;
 
CREATE SOURCE BinarySource using FileReader (
  directory:'Samples',
  wildcard:'test.bin',
  positionByEOF:false
)
PARSE USING BinaryParser (
  metadata:'Samples/metadata.json',	
  endian:true
)
OUTPUT TO BinaryStream;

CREATE TARGET BinaryDump USING LogWriter(
  name:BinaryReader, 
  filename:'out.log'
) INPUT FROM BinaryStream;

CREATE TYPE OrderType (
  OrderId integer,
  DistrictId integer,
  WarehouseId integer,
  Name String,
  CountId integer,
  Price double);
     
CREATE STREAM OrderStream OF OrderType;

CREATE CQ OrderCQ 
INSERT into OrderStream
SELECT data[0],
  data[1],
  data[2],
  data[3],
  data[4],
  TO_DOUBLE(data[5])
FROM BinaryStream;

CREATE TARGET DSVOut
USING FileWriter(filename: binary_output)
FORMAT USING DSVFormatter()
INPUT FROM OrderStream;

END APPLICATION bin;

The binary_output.csv output file will start like this:

10000,0,100,John 0,7,243.53
10001,1,101,John 1,73,2539.67
10002,2,102,John 2,130,4522.7
10003,3,103,John 3,144,5009.76
10004,4,104,John 4,123,4279.17
10005,5,105,John 5,40,1391.6