/*
   the user did not specify regions, print all alignments from all input-files
   this is strategy #1 to do this, create a ref_iter for every reference each
    + ... less cursors will be open at the same time, more resource efficient
    - ... if more than one input-file, the output will be sorted only within each reference
*/


/*
   the user did not specify regions, print all alignments from all input-files
   this is strategy #2 to do this, throw all iterators for all input-files and all there references
   into one set_iter.
    + ... if more than one input-file, everything will be sorted
    - ... creates a large number of open cursors ( because of sub-cursors due to schema-functions )
          this can result in not beeing able to perform the functions at all because of running out of resources
    - ... a long delay at start up, before the 1st alignment is printed ( all the cursors have to be opened )
*/


/*
   the user has specified certain regions on the references,
   print only alignments, that start in these regions
*/


/*
   this is called from sam-dump3.c, it prepares the iterators and then walks them
   ---> only entry into this module <---
*/


/**********************************************************************************
    0x001   template having multiple fragments in sequencing
            (row->nreads > 1 )

    0x002   each fragment properly aligned according to the aligner
            ( never the case in an unaligned read )

    0x004   fragment unmapped
            ( always true in an unaligned read )

    0x008   next fragment in the template unmapped
            ( is the mate aligned? next: read_idx+1, if read_idx==row->nreads-1 then read_idx-1 )

    0x010   SEQ being reverse complemented
            ( column 'READ_TYPE' has bit READ_TYPE_REVERSE set )

    0x020   SEQ of the next fragment in the template being reversed
            ( is the mate reversed? )

    0x040   the first fragment in the template
            ( read_idx == 0 )

    0x080   the last fragment in the template
            ( read_idx == ( row->nreads - 1 ) )

    0x100   secondary alignment
            ( never the case in an unaligned read )

    0x200   not passing quality controls
            ( column 'READ_FILTER' has bit READ_FILTER_REJECT set )

    0x400   PCR or optical duplicate
            ( column 'READ_FILTER' has bit READ_FILTER_CRITERIA set )
**********************************************************************************/


/* called if we are dumping from a legacy table instead from a database */


/* we are printing from a sra-database, we print all unaligned read we can find */


/* we are printing from a (legacy) table not from a database! */


/* entry point from sam-dump3.c */


/* helper func to detect if the given reference_region has ranges to be skiped */


/* helper to create a skiplist-node, walk the given the ref-region fo find and enter all skip positions */


/* SQ-lines have to be uniue by the SN-tag */
/* RG-lines have to be uniue by the ID-tag */


/* gap contains the indices of the wobbles in op
 * gap[0] is between read 1 and 2; it is the 'B'
 * gap[1] is between read 2 and 3; it is an 'N'
 * gap[2] is between read 3 and 4; it is an 'N'
 */


/*
static rc_t get_int64_option( Args * args, const char * name, uint64_t def, uint64_t * value ) {
    const char * s;
    rc_t rc = get_str_option( args, name, &s );
    *value = def;
    if ( rc == 0 && s != NULL ) {
        char *endp;
        *value = strtou32( s, &endp, 0 );
        if ( *endp != '\0' ) {
            rc = RC( rcExe, rcArgv, rcProcessing, rcParam, rcInvalid );
            (void)PLOGERR( klogErr, ( klogErr, rc, "error converting string to integer option '$(t)'", "t=%s", name ) );
        } else if ( *value == 0 ) {
            *value = def;
        }
    }
    return rc;
}
*/


/*********************************************************************************************

                                            fa ... fully aligned reads
                                            ha ... half aligned reads
                                            hu ... half unaligned reads
                                            fu ... fully unaligned reads

    regions > 0     |   dump-unaligned  |
    ----------------------------------------------------------------------
        no          |       no          |   fa  ha  hu
        no          |       yes         |   fa  ha  hu  fu
        yes         |       no          |   fa  ha
        yes         |       yes         |   fa  ha  hu

*********************************************************************************************/


/****************************************************************************************
    splits an argument

    example: "/path/file=grp1" into path = "/path/file" and attribute = "grp1"
    or
    example: "/path/file" into path = "/path/file" and attribute = NULL

****************************************************************************************/


/* try to find the key in the expanded path */


/* try to find the key in the child-vector of root ( does expand/un-expand top-level drop-down menus ) */


/*
static bool is_the_selected_node_expanded( KTUI_Menu_Node * node )
{
    bool res = false;
    if ( node != NULL && node->expanded )
    {
        KTUI_Menu_Node * sub = VectorGet ( &node->children, node->selected );
        if ( sub != NULL )
            res = sub->expanded;
    }
    return res;
}
*/


/* the attribute and color of regular text */


/* the attribute and color of highlighted text */


/* called if the value of a widged has changed */


/* if max VK_* constant value is greater than 0xFF, then this code must be revised */


/* Return the current wall-clock time */


/*
** Begin timing an operation
*/


/* Return the difference of two time_structs in seconds */


/*
** Print the timing results.
*/


/*
** Check to see if we have timer support.  Return 1 if necessary
** support found (or found previously).
*/


/* Return the difference of two FILETIME structs in seconds */


/*
** Determines if a string is a number of not.
*/


/*
** Compute a string length that is limited to what can be stored in
** lower 30 bits of a 32-bit signed integer.
*/


/*
** This routine reads a line of text from FILE in, stores
** the text in memory obtained from malloc() and returns a pointer
** to the text.  NULL is returned at end of file, or if malloc()
** fails.
**
** If zLine is not NULL then it is a malloced buffer returned from
** a previous call to this routine that may be reused.
*/


/*
** Retrieve a single line of input text.
**
** If in==0 then read from standard input and prompt before each line.
** If isContinuation is true, then a continuation prompt is appropriate.
** If isContinuation is zero, then the main prompt should be used.
**
** If zPrior is not NULL then it is a buffer from a prior call to this
** routine that can be reused.
**
** The result is stored in space obtained from malloc() and must either
** be freed by the caller or else passed back into this routine via the
** zPrior argument for reuse.
*/


/*
** A callback for the sqlite3_log() interface.
*/


/*
** Output the given string as a hex-encoded blob (eg. X'1234' )
*/


/*
** Output the given string as a quoted string using SQL quoting conventions.
*/


/*
** Output the given string as a quoted according to C or TCL quoting rules.
*/


/*
** Output the given string with characters that are special to
** HTML escaped.
*/


/*
** Output a single term of CSV.  Actually, p->colSeparator is used for
** the separator, which may or may not be a comma.  p->nullValue is
** the null value.  Strings are quoted if necessary.  The separator
** is only issued if bSep is true.
*/


/*
** This routine runs when the user presses Ctrl-C
*/


/*
** When the ".auth ON" is set, the following authorizer callback is
** invoked.  It always returns SQLITE_OK.
*/


/*
** Print a schema statement.  Part of MODE_Semi and MODE_Pretty output.
**
** This routine converts some CREATE TABLE statements for shadow tables
** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements.
*/


/*
** This is the callback routine that the shell
** invokes for each row of a query result.
*/


/*
** This is the callback routine that the SQLite library
** invokes for each row of a query result.
*/


/* zIn is either a pointer to a NULL-terminated string in memory obtained
** from malloc(), or a NULL pointer. The string pointed to by zAppend is
** added to zIn, and the result returned in memory obtained from malloc().
** zIn, if it was not NULL, is freed.
**
** If the third argument, quote, is not '\0', then it is used as a
** quote character for zAppend.
*/


/*
** Execute a query statement that will generate SQL output.  Print
** the result columns, comma-separated, on a line and then add a
** semicolon terminator to the end of that line.
**
** If the number of columns is 1 and that column contains text "--"
** then write the semicolon on a separate line.  That way, if a
** "--" comment occurs at the end of the statement, the comment
** won't consume the semicolon terminator.
*/


/*
** Allocate space and save off current error string.
*/


/*
** Attempt to display I/O stats on Linux using /proc/PID/io
*/


/*
** Display memory stats.
*/


/*
** Display scan stats.
*/


/*
** Parameter azArray points to a zero-terminated array of strings. zStr
** points to a single nul-terminated string. Return non-zero if zStr
** is equal, according to strcmp(), to any of the strings in the array.
** Otherwise, return zero.
*/


/*
** If compiled statement pSql appears to be an EXPLAIN statement, allocate
** and populate the ShellState.aiIndent[] array with the number of
** spaces each opcode should be indented before it is output.
**
** The indenting rules are:
**
**     * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent
**       all opcodes that occur between the p2 jump destination and the opcode
**       itself by 2 spaces.
**
**     * For each "Goto", if the jump destination is earlier in the program
**       and ends on one of:
**          Yield  SeekGt  SeekLt  RowSetRead  Rewind
**       or if the P1 parameter is one instead of zero,
**       then indent all opcodes between the earlier instruction
**       and "Goto" by 2 spaces.
*/


/*
** Free the array allocated by explain_data_prepare().
*/


/*
** Run a prepared statement
*/


/*
** Execute a statement or set of statements.  Print
** any result rows/columns depending on the current mode
** set via the supplied callback.
**
** This is very similar to SQLite's built-in sqlite3_exec()
** function except it takes a slightly different callback
** and callback data argument.
*/


/*
** This is a different callback routine used for dumping the database.
** Each row received by this callback consists of a table name,
** the table type ("index" or "table") and SQL to create the table.
** This routine should print text sufficient to recreate the table.
*/


/*
** Run zQuery.  Use dump_callback() as the callback routine so that
** the contents of the query are output as SQL statements.
**
** If we get a SQLITE_CORRUPT error, rerun the query after appending
** "ORDER BY rowid DESC" to the end.
*/


/*
** Print help information for the ".sessions" command
*/


/*
** Read the content of file zName into memory obtained from sqlite3_malloc64()
** and return a pointer to the buffer. The caller is responsible for freeing
** the memory.
**
** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
** read.
**
** For convenience, a nul-terminator byte is always appended to the data read
** from the file before the buffer is returned. This byte is not included in
** the final value of (*pnByte), if applicable.
**
** NULL is returned if any error is encountered. The final value of *pnByte
** is undefined in this case.
*/


/*
** Implementation of the "readfile(X)" SQL function.  The entire content
** of the file named X is read and returned as a BLOB.  NULL is returned
** if the file does not exist or is unreadable.
*/


/*
** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
** is written into file X.  The number of bytes written is returned.  Or
** NULL is returned if something goes wrong, such as being unable to open
** file X for writing.
*/


/*
** Close a single OpenSession object and release all of its associated
** resources.
*/


/*
** Make sure the database is open.  If it is not, then open it.  If
** the database fails to open, print an error message and exit.
*/


/*
** Do C-language style dequoting.
**
**    \a    -> alarm
**    \b    -> backspace
**    \t    -> tab
**    \n    -> newline
**    \v    -> vertical tab
**    \f    -> form feed
**    \r    -> carriage return
**    \s    -> space
**    \"    -> "
**    \'    -> '
**    \\    -> backslash
**    \NNN  -> ascii character NNN in octal
*/


/*
** Return the value of a hexadecimal digit.  Return -1 if the input
** is not a hex digit.
*/


/*
** Interpret zArg as an integer value, possibly with suffixes.
*/


/*
** Interpret zArg as either an integer or a boolean value.  Return 1 or 0
** for TRUE and FALSE.  Return the integer value if appropriate.
*/


/*
** Try to open an output file.   The names "stdout" and "stderr" are
** recognized and do the right thing.  NULL is returned if the output
** filename is "off".
*/


/*
** A routine for handling output from sqlite3_trace().
*/


/* Append a single byte to z[] */


/*
** Try to transfer data for table zTable.  If an error is seen while
** moving forward, try to go backwards.  The backwards movement won't
** work for WITHOUT ROWID tables.
*/


/*
** Try to transfer all rows of the schema that match zWhere.  For
** each row, invoke xForEach() on the object defined by that row.
** If an error is encountered while moving forward through the
** sqlite_master table, try again moving backwards.
*/


/*
** Open a new database file named "zNewDb".  Try to recover as much information
** as possible out of the main database (which might be corrupt) and write it
** into zNewDb.
*/


/*
** Change the output file back to stdout
*/


/*
** Run an SQL command and return the single integer result.
*/


/*
** Implementation of the ".info" command.
**
** Return 1 on error, 2 to exit, and 0 otherwise.
*/


/*
** Print the current sqlite3_errmsg() value to stderr and return 1.
*/


/*
** Compare the pattern in zGlob[] against the text in z[].  Return TRUE
** if they match and FALSE (0) if they do not match.
**
** Globbing rules:
**
**      '*'       Matches any sequence of zero or more characters.
**
**      '?'       Matches exactly one character.
**
**     [...]      Matches one character from the enclosed list of
**                characters.
**
**     [^...]     Matches one character not in the enclosed list.
**
**      '#'       Matches any sequence of one or more digits with an
**                optional + or - sign in front
**
**      ' '       Any span of whitespace matches any other span of
**                whitespace.
**
** Extra whitespace at the end of z[] is ignored.
*/


/*
** Compare the string as a command-line option with either one or two
** initial "-" characters.
*/


/*
** Delete a file.
*/


/*
** The implementation of SQL scalar function fkey_collate_clause(), used
** by the ".lint fkey-indexes" command. This scalar function is always
** called with four arguments - the parent table name, the parent column name,
** the child table name and the child column name.
**
**   fkey_collate_clause('parent-tab', 'parent-col', 'child-tab', 'child-col')
**
** If either of the named tables or columns do not exist, this function
** returns an empty string. An empty string is also returned if both tables
** and columns exist but have the same default collation sequence. Or,
** if both exist but the default collation sequences are different, this
** function returns the string " COLLATE <parent-collation>", where
** <parent-collation> is the default collation sequence of the parent column.
*/


/*
** The implementation of dot-command ".lint fkey-indexes".
*/


/*
** Implementation of ".lint" dot command.
*/


/*
** Return TRUE if a semicolon occurs anywhere in the first N characters
** of string z[].
*/


/*
** Test to see if a line consists entirely of whitespace.
*/


/*
** Return TRUE if the line typed in is an SQL command terminator other
** than a semi-colon.  The SQL Server style "go" command is understood
** as is the Oracle "/".
*/


/*
** Return true if zSql is a complete SQL statement.  Return false if it
** ends in the middle of a string literal or C-style comment.
*/


/*
** Run a single line of SQL
*/


/*
** Read input from *in and process it.  If *in==0 then input
** is interactive - the user is typing it it.  Otherwise, input
** is coming from a file or device.  A prompt is issued and history
** is saved only if input is interactive.  An interrupt signal will
** cause this routine to exit immediately, unless input is interactive.
**
** Return the number of errors.
*/


/*
** Return a pathname which is the user's home directory.  A
** 0 return indicates an error of some kind.
*/


/*
** Read input from the file given by sqliterc_override.  Or if that
** parameter is NULL, take input from ~/.sqliterc
**
** Returns the number of errors.
*/


/*
** Get the argument to an --option.  Throw an error and die if no argument
** is available.
*/


/********************************************************************
helper function to display the version of the vdb-manager
********************************************************************/


/********************************************************************
helper function to test if a given path is a vdb-table
********************************************************************/


/********************************************************************
helper function to test if a given path is a vdb-column
by testing if the parent/parent dir is a vdb-table
********************************************************************/


/********************************************************************
helper function to test if a given path is a vdb-database
********************************************************************/


/*************************************************************************************
helper-function to extract the name of the first table of a database
and put it into the dump-context
*************************************************************************************/


/*************************************************************************************
helper-function to check if a given table is in the list of tables
if found put that name into the dump-context
*************************************************************************************/


/* implementation of the dimension-translation-functions */


/* a single column-definition */


/* a vector of column-definitions */


/*************************************************************************************
n_bits   [IN] ... number of bits

creates a bitmask to mask exactly the given number of bits from a longer value
*************************************************************************************/


/*************************************************************************************
src         [IN] ... buffer containing the data
dpo         [IN] ... pointer to buffer-offset (bit-offset-part will be ignored)

dumps an ascii-string
*************************************************************************************/


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

dumps boolean values (up to 64 bits)
*************************************************************************************/


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

unsigned int's up to a length of 64 bit are supported
unused bits are masked out
*************************************************************************************/


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

signed int's up to a length of 64 bit are supported
unused bits are masked out
*************************************************************************************/


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

only 2 float-types: "float"(32 bit) and "double"(64 bit) are supported
*************************************************************************************/


/* special function to translate dim=2,bits=1 into a DNA-base */


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

dumps one data-element (or a vector of it)
*************************************************************************************/


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

dumps all data-elements of a cell
*************************************************************************************/


/* unsigned integer values, but we are not on a byte-boundary
  called by vdt_format_slice_nbb below */


/* signed integer values, but we are not on a byte-boundary
  called by vdt_format_slice_nbb below */


/* here the bit-size should be only 32-bit or 64-bit, but we are not on a byte-boundary
  called by vdt_format_slice_nbb below */


/* here the bit-size should be only 8-bit, but we are not on a byte-boundary
  called by vdt_format_slice_nbb below */


/* not on a byte-boundary, 1 dimensional array of values
 called by: vdt_format_cell_nbb_dim2_v2 AND vdt_format_cell_v2 below */


/* not on a byte-boundary, 2 dimensional array of values ( grouped ), translate each value, json-format
 called by vdt_format_cell_nbb_dim2_v2 below */


/* not on a byte-boundary, 2 dimensional array of values ( grouped ), translate each value, dflt-format
 called by vdt_format_cell_nbb_dim2_v2 below */


/* not on a byte-boundary, 2 dimensional array of values ( grouped )
 called by vdt_format_cell_v2 below */


/* on a byte-boundary, slice of ascii-test */


/* on a byte-boundary, 1 dimensional array of values */


/* on a byte-boundary, 2 dimensional array of values ( grouped ) translated to txt, jsond-format
 called by by vdt_format_cell_bb_dim2_v2() below */


/* on a byte-boundary, 2 dimensional array of values ( grouped ) translated to txt, dflt-format
 called by vdt_format_cell_bb_dim2_v2() below */


/* on a byte-boundary, 2 dimensional array of values ( grouped )
 called by vdt_format_cell_v2 below */


/*************************************************************************************
src         [IN] ... buffer containing the data
my_col_def  [IN] ... the definition of the column to be dumped

new and improved print of a cell, takes advantage of the fact that most ( if not
all ) cells have bit-offset == 0 and can be printed as a typecast to an array
called by vdm_read_cell_data() in vdb-dump.c
*************************************************************************************/


/********************************************************************
the dump context contains all informations needed to execute the dump
********************************************************************/


/* not static because can be called directly from vdb-dump.c */


/*************************************************************************************
    dump_tab_table:
    * called by "dump_db_table()" and "dump_tab()" as a fkt-pointer
    * opens a cursor to read
    * checks if the user did not specify columns, or wants all columns ( "*" )
        no columns specified ---> calls "col_defs_extract_from_table()"
        columns specified ---> calls "col_defs_parse_string()"
    * we end up with a list of column-definitions (name,type) in col_defs
    * calls "col_defs_add_to_cursor()" to add them to the cursor
    * opens the cursor
    * calls "dump_rows()" to execute the dump
    * destroys the col_defs - structure
    * releases the cursor

ctx [IN] ... contains path, tablename, columns, row-range etc.
tbl [IN] ... open table needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    vdm_dump_opened_view:
    * opens a cursor to read from a view

ctx [IN] ... contains path, tablename, columns, row-range etc.
view [IN] ... open view needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    dump_db_table:
    * called by "dump_database()" as a fkt-pointer
    * opens a table to read
    * calls "dump_tab_table()" to do the dump
    * releases the table

ctx  [IN] ... contains path, tablename, columns, row-range etc.
db   [IN] ... open database needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    dump_the_tab_schema:
    * called by "dump_the_db_schema()" and "dump_table()" as a function pointer
    * opens the schema of a table
    * calls "VSchemaDump()" to dump this schema
    * releases the schema

ctx   [IN] ... contains path, tablename, columns, row-range etc.
tbl   [IN] ... open table needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    dump_the_db_schema:
    * called by "dump_database()" as a function pointer
    * opens a table to read
    * calls "dump_the_tab_schema()" to dump the schema of this table
    * releases the table

ctx   [IN] ... contains path, tablename, columns, row-range etc.
db    [IN] ... open database needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    print_column_info:
    * get's called from "enum_tab_columns()" via VectorForEach
    * prints: table-name, col-idx, bits, dimension, domain and name
      for every column

item  [IN] ... pointer to a column-definition-struct ( from "vdb-dump-coldefs.h" )
data  [IN] ... pointer to dump-context ( in this case to have a line-idx )
*************************************************************************************/


/*************************************************************************************
    enum_tab_columns:
    * called by "enum_db_columns()" and "dump_table()" as fkt-pointer
    * initializes a column-definitions-structure
    * calls col_defs_extract_from_table() to enumerate all columns in the table
    ( both functions are in "vdb-dump-coldefs.h" )
    * loops through the columns and prints it's information
    * releases the column-definitions-structure

ctx   [IN] ... contains path, tablename, columns, row-range etc.
tbl   [IN] ... open table needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    enum_db_columns:
    * called by "dump_database()" as fkt-pointer
    * opens the table
    * calls enum_tab_columns()
    * releases table

ctx   [IN] ... contains path, tablename, columns, row-range etc.
db    [IN] ... open database needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    print_db_id_range:
    * called by "dump_database()" as fkt-pointer
    * opens the table
    * calls print_tab_id_range()
    * releases table

ctx   [IN] ... contains path, tablename, columns, row-range etc.
db    [IN] ... open database needed for vdb-calls
*************************************************************************************/


/*************************************************************************************
    dump_tab_fkt:
    * called by "dump_table()"
    * if the user has provided the name of a schema-file:
        - make a schema from manager
        - parse the schema-file
        - use this schema to open the database
    * if we do not have a schema-file, use NULL ( that means the internal tab-schema )
    * open the table for read
    * call the function-pointer, with context and table
    * release table and schema

ctx    [IN] ... contains source-path, tablename, columns and row-range as ascii-text
db_fkt [IN] ... function to be called if directory, manager and database are open
*************************************************************************************/


/***************************************************************************
    dump_table:
    * called by "dump_main()" to handle a table
    * calls the dump_tab_fkt with 3 different fkt-pointers as argument
      depending on what was selected at the commandline

ctx  [IN] ... contains path, tablename, columns, row-range etc.
mgr  [IN] ... open manager needed for vdb-calls
***************************************************************************/


/*************************************************************************************
    dump_db_fkt:
    * called by "dump_database()" to handle a database
    * if the user has provided the name of a schema-file:
        - make a schema from manager
        - parse the schema-file
        - use this schema to open the database
    * if we do not have a schema-file, use NULL ( that means the internal db-schema )
    * open the database for read
    * if the user has not given a table-name to dump:
        - take the first table found in the database
    * if we now have a table-name
        - call the function-pointer, with context and database
    * if we do not have a table-name
        - request display of the usage-message
    * release database and schema

ctx    [IN] ... contains source-path, tablename, columns and row-range as ascii-text
db_fkt [IN] ... function to be called if directory, manager and database are open
*************************************************************************************/


/***************************************************************************
    dump_database:
    * called by "dump_main()"
    * calls the dump_db_fkt with 4 different fkt-pointers as argument
      depending on what was selected at the commandline

ctx   [IN] ... contains path, tablename, columns, row-range etc.
mgr   [IN] ... open manager needed for vdb-calls
***************************************************************************/


// returns a VCursor* in ctx->cursor; may be NULL if the view has no data


/***************************************************************************
    dump_unbound_view:
    * called by "dump_main()" to handle a view
    * similar to dump_table but dumps a view
    * before dumping, binds the view's arguments to existing tables according to the view-spec string given in ctx.view

ctx        [IN] ... contains path, view instantiation string, columns, row-range etc.
my_manager [IN] ... open manager needed for vdb-calls
***************************************************************************/


/* use the KSORT macro so the compiler can optimize everything */


/* referential integrity and data checks for sequence, primary and secondary alignment tables */


/* database referential integrity check for alignment database */


/* ### Common splitter factory code ##################################################### */


/* ### Common dumper code ##################################################### */


/* main entry point of the file */


/* filter out non-bio reads */


/* filter out reads by leading/trialing quality */


/* if all active reads len >= minreadlen, reads are splitted into separate files, otherwise on single key "" is returnded for all reads */


/* refactored June 13 2014 by Wolfgang to fix a mysterious bug on windows where b_qseq resolved into false on windows
   and true on posix! The original code was this:

    if( self == NULL ) {
          rc = RC(rcSRA, rcType, rcConstructing, rcParam, rcNull);
      } else if( (rc = IlluminaReaderMake(&self->reader, self->table, self->accession,
                                  IlluminaArgs.opt & eRead, IlluminaArgs.opt & (eQual1_S | eQual1_M), IlluminaArgs.opt & eQual4,
                                  IlluminaArgs.opt & eIntensity, IlluminaArgs.opt & eNoise, IlluminaArgs.opt & eSignal,
                                  IlluminaArgs.opt & (eQSeq_S | eQSeq_M), 0, 0)) == 0 ) {
          rc = KDataBufferMakeBytes(&self->buf, DATABUFFERINITSIZE);
        }

    Anton's old, more dense version should do the same, but it does not on Windows!

    eQSeq_M is an enum, defined at the top of this file to have the value 0x100.
    It will be type-casted into bool because that is the type IlluminaReaderMake() wants.
    ( bool )eQSeq_M is true for GCC/LLVM and false for the MS-compiler!
    Casting enum's directly without comparison into boolean does not work for the MS-compiler.
*/


/* we have to test if argv[0] ends in 'fastq-dump' or 'fastq-dump.exe',
   and we have an argument called '--fasta' */


/* leave only up to 2 first bio reads */


/* skip all reads with len < minreadlen */


/* new function to split a path into directory-filename aka stem and extension (Dec 2024) */


/* ------------------------------------------
   if qual-defline is NULL --> FASTA, else FASTQ
   if both def-lines are NULL --> pick default one's
   ------------------------------------------ */


/* submit a buffer to the multi-writer ( via a queue into a different thread! ) */


/* used by temp_registry.c */


/* read_id_0 means read-id zero based */


/* we have NO output-dir and NO output-file */


/* we have an output-dir and NO output-file */


/* it is a cSRA with alignments! */


/* it is a database without alignments! */


/* it is a flat table! */


/* just like perform_fastq_whole_spot_join(), but without quality and calling print_fasta_1_read() */


/* called from the background-thread */


/* create a var-name-element, makes a copy of the given string and owns that copy */


/* create a literal format-element, makes a copy of the given string and owns that copy */


/* create a string format-element, stores the index of the string to use */


/*we need both: str-args AND int-args, because of the alternative idx-usage */


/* releases an element, data-pointer to match VectorWhack-callback */


/* create an empty var-print struct, to be added into later */


/* create a var-print struct and fill it with var_fmt_t - elements */


/* release a var-print struct and call destructors on its elements */


/* apply the var-fmt-struct to the given arguments, write result to buffer */


/* apply the var-fmt-struct to the given arguments, print the result to file */


/*
 * determine the archive type for KFile f with pathname name
 *
 * This could be extended to handle tar files with a larger head size and
 * and some less simple checks for a tar header block at the start of
 * the file.
 */


/*
 * Copy a file from a const KFile * to a KFile * with the paths for the two
 * for logging purposes
 *
 * return rc_t = 0 for success
 * return rc_t != 0 for failure
 */


/*
 * determine the encryption type for KFile f with pathname name
 */


/*
 * Check a file path name for ending in the extension used by this program
 *
 * return true if it ends with the extension and false if it does not
 */


/*
** In this version of BtreeMoveto, pKey is a packed index record
** such as is generated by the OP_MakeRecord opcode.  Unpack the
** record and then call sqlite3BtreeIndexMoveto() to do the work.
*/


/*
** The cell pCell is currently part of page pSrc but will ultimately be part
** of pPage.  (pSrc and pPage are often the same.)  If pCell contains a
** pointer to an overflow page, insert an entry into the pointer-map for
** the overflow page that will be valid after pCell has been moved to pPage.
*/


/*
** This routine is called after all opcodes have been inserted.  It loops
** through all the opcodes and fixes up some details.
**
** (1) For each jump instruction with a negative P2 value (a label)
**     resolve the P2 value to an actual address.
**
** (2) Compute the maximum number of arguments used by any SQL function
**     and store that value in *pMaxFuncArgs.
**
** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately
**     indicate what the prepared statement actually does.
**
** (4) (discontinued)
**
** (5) Reclaim the memory allocated for storing labels.
**
** This routine will only function correctly if the mkopcodeh.tcl generator
** script numbers the opcodes correctly.  Changes to this routine must be
** coordinated with changes to mkopcodeh.tcl.
*/


/*
** Free all memory associated with the Vdbe passed as the second argument,
** except for object itself, which is preserved.
**
** The difference between this function and sqlite3VdbeDelete() is that
** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with
** the database connection and frees the object itself.
*/


/*
** This function is called to generate code executed when a row is deleted
** from the parent table of foreign key constraint pFKey and, if pFKey is
** deferred, when a row is inserted into the same table. When generating
** code for an SQL UPDATE operation, this function may be called twice -
** once to "delete" the old row and once to "insert" the new row.
**
** Parameter nIncr is passed -1 when inserting a row (as this may decrease
** the number of FK violations in the db) or +1 when deleting one (as this
** may increase the number of FK constraint problems).
**
** The code generated by this function scans through the rows in the child
** table that correspond to the parent table row being deleted or inserted.
** For each child row found, one of the following actions is taken:
**
**   Operation | FK type   | Action taken
**   --------------------------------------------------------------------------
**   DELETE      immediate   Increment the "immediate constraint counter".
**
**   INSERT      immediate   Decrement the "immediate constraint counter".
**
**   DELETE      deferred    Increment the "deferred constraint counter".
**
**   INSERT      deferred    Decrement the "deferred constraint counter".
**
** These operations are identified in the comment at the top of this file
** (fkey.c) as "I.2" and "D.2".
*/


/*
** Handle the special case of a compound-select that originates from a
** VALUES clause.  By handling this as a special case, we avoid deep
** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT
** on a VALUES clause.
**
** Because the Select object originates from a VALUES clause:
**   (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1
**   (2) All terms are UNION ALL
**   (3) There is no ORDER BY clause
**
** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES
** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))").
** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case.
** Since the limit is exactly 1, we only need to evaluate the left-most VALUES.
*/


/* If the begin parameter was set check if the rowid is equal or above it */


/* If the end parameter was set check if the rowid is equal or below it */


/*
 * out is a pre-allocated buffer.
 */


/*
 * copies top/inname (a directory)
 * to targettop/outname, i.e. creates outname as a copy of that directory.
 */


/** @brief Apply the rules to determine if a read should be filtered
 **/


/** length of a common prefix
 */


/** check if cachePath == inPath + ".vdbcache"
 */


/* buffered recursion entrypoint */


/* -----
 * copycat_add_enc
 *
 * add an encryptor to the copy chain
 */


/* -----
 * copycat_add_sz
 *
 * if we are encrypting the output we need to do an actual byte count across the
 * write side of the encryption so we add a filter into the copy chain.  If we
 * are not encrypting we do this calculation in the ccat chain for the outside file
 * and never reach this function.
 */


/* -----
 * copycat_add_md5
 *
 * if we are encrypting the output we need to do an md5 calculation across the
 * write side of the encryption so we add a filter into the copy chain.  If we
 * are not encrypting we do the md5 calculation in the ccat chain for the outside file
 * and never reach this function.
 */


/* -----
 * copycat_add_crc
 *
 * this is the first function in building the write side of the copy chain
 *
 * We calculate a crc on the outgoing file and none of the interior files
 * and we do that at this point in the chain regardless of whether we will
 * encrypt the outgoing file.
 *
 * We'll add the crc calculator to the write side stream and then decide
 * whether to add an encryptor into the chain.  We add one to the write side
 * of the chain if we have an encrypting password or jump to finishing the
 * copy chain if we do not
 */


/* -----
 * copycat_add_buf2
 *
 * This function adds buffering to the outermostDecrypted file read so that we
 * can read the first portion of the file for file type analysis and then
 * re-read it for cataloging.
 *
 * At this point the only file type analysis needed is to determine
 * whether we have in incoming encrypted file We've already checked for
 * the existence of the key or we wouldn't get here.
 */


/* -----
 * copycat_add_dec
 *
 * add a decryption to the read side of the copy chain.
 * no other decisions made here
 */


/* -----
 * copycat_add_dec
 *
 * At this point the only file type analysis needed is to determine
 * whether we have in incoming encrypted file We've already checked for
 * the existence of the key or we wouldn't get here.
 */


/* -----
 * copycat
 *
 * The copycat function is the actual copy and catalog function.
 * All before this function is called is building toward this.
 *
 * The functions prefixed copycat_add_ are functions used in building the
 * chain of KFS and other filters for doing the copy function.  Some
 * cataloging is included where the output file is encoded and to
 * generate the CRC unique to the outer file in the catalog.
 * The ccat_add_ functions are filters toward the leafs in the catlog
 * portion of the program.
 *
 * Much of this could be inlined but the indention creep across the screen
 * and the handling of levels of indention would get intense.  Also some
 * filters are skipped which also would have made it more unweildy to
 * write and maintain.
 *
 * In this function we build a node for cataloging the outermost layer of
 * the output file which may differ from the file as read here if a
 * decryption and/or encryption is added.
 *
 * We then
 */


/** Note: buffer must be at least 8 bytes
 */


/** Note: buffer must be at least 4 bytes
 */


/** Scans buffer for FASTQ format compliance
 */


/*--------------------------------------------------------------------------
 * CCFileNode
 *  a node with a size and modification timestamp
 *  has a file type determined by magic/etc.
 *  has an md5 checksum
 *
 *  how would an access mode be used? access mode of a file is
 *  whatever the filesystem says it is, and within an archive,
 *  it's read-only based upon access mode of outer file...
 */


/* Make
 *  creates an object with provided properties
 *  md5 digest needs to be filled in afterward
 */


/* AddChunk
 *  adds a chunk to the chunk file
 */


/* Make
 *  creates a cached file wrapper
 */


/* Make
 *  creates a symlink object
 */


/* Make
 *  make a node name
 */


/* Make
 *  creates an archive object
 */


/* Make
 *  make a root tree or sub-directory
 */


/* Insert
 *  create an entry into a tree
 *  parses path into required sub-directories
 */


/* Find
 *  find a named node
 *  returns NULL if not found
 */


/* Link
 *  create a symlink to existing node
 */


/* ======================================================================
 * return the RFC 2045 base 64 value for a byte character
 * -1 for any out of range
 *
 * base 64 uses A-Z as 0-25, a-z as 26-51, 0-9 as 52-61, + as 62 and / as 63
 */


/* ======================================================================
 * return the RFC 2045 base 64 value of a string at <str> of length <len>
 *
 * Interpreted the specification to mean that you simply ignore any out
 * of range characters.  They are not treated as bytes of 0 as that would
 * mean a shift of earlier bytes.
 *
 * Each character is 6 bits of the final value.
 */


/* ======================================================================
 * tar_strtoll
 *
 * This function will convert a string in a tar header into a 64 signed
 * integer.
 *
 * The original tar header used just octal numbers in ascii in fixed length
 * fields.  As normal in the computer software world these "obviously plenty
 * big" fields became way too small.
 *
 * GNU tar invented two approaches to make these numeric fields "bigger" but
 * only prolonged the pain by squeezing the new numbers into the same fields.
 *
 * In the tar v7 and ustar based tar headers the fields are:
 *	Name	Size	Octal Range	Interpretation
 *	mode	8	0-2097151 	bit flags
 *	uid	8	0-2097151
 *	gid	8	0-2097151
 *	size	12	0-8589934591	up to 8 Giga-Byte files
 *	mtime	12	0-8589934591 	1970/01/01 00:00:00 GMT-2242/03/16 12:56:31 GMT
 *
 *	atime	12	0-8589934591 	1970/01/01 00:00:00 GMT-2242/03/16 12:56:31 GMT
 *	ctime	12	0-8589934591 	1970/01/01 00:00:00 GMT-2242/03/16 12:56:31 GMT
 *
 * GNU's first and already obsolete approach was to put Mime base-64 numbers
 * With these the first byte is '+' or '-' to signal it isn't octal ASCII.
 *
 * GNU's second approach is base-256 which is a big endian binary string of lengths other
 * than the 1, 2, 4 or 8 bytes of the standard integer types.  In this approach the first
 * byte is either 0x80 for a positive number or 0xFF for a negative number.
 */


/* ======================================================================
 * tar_header_type
 * determine most probable tar header block type
 */


/* ======================================================================
 * local module-wide variables (reduction in parameter pushing)
 *
 * Module shared variables; kinda like C++ class elements
 */


/* ----------
 * The mtime and atime <value> is defined as
 *	The pax utility shall write an mtime record for each file in write
 *	or copy modes if the file's modification time cannot be represented
 *	exactly in the ustar header logical record described in ustar
 *	Interchange Format. This can occur if the time is out of ustar range,
 *	or if the file system of the underlying implementation supports
 *	non-integer time granularities and the time is not an integer. All of
 *	these time records shall be formatted as a decimal representation of
 *	the time in seconds since the Epoch. If a period ( '.' ) decimal
 *	point character is present, the digits to the right of the point shall
 *	represent the units of a subsecond timing granularity, where the first
 *	digit is tenths of a second and each subsequent digit is a tenth of
 *	the previous digit. In read or copy mode, the pax utility shall
 *	truncate the time of a file to the greatest value that is not greater
 *	than the input header file time. In write or copy mode, the pax
 *	utility shall output a time exactly if it can be represented exactly
 *	as a decimal number, and otherwise shall generate only enough digits
 *	so that the same time shall be recovered if the file is extracted on a
 *	system whose underlying implementation supports the same time
 *	granularity.
 *
 * So...
 * Looks like for our purposes we ignore anything after a possible decimal
 * point and just use the integer part of whatever is there and just not be
 * to worried about it since we aren't creating any of this, just reading ie.
 */


/* ======================================================================
 *
 * offset is the byte position within the tar file
 * hard_limit is the byte position with in the tar file that is not mapped
 *
 * This is the ugliest function/method in the whole module.  The extensions
 * to the tar header are not done in a consistent manner so convolutions
 * have to be made to support all manner of extensions.
 */


/* ----------------------------------------------------------------------
 * CCFileMake
 *  create a new file object
 */


/* ----------
 * Constructor/initializer for a buffer: called only from a BufferMgr
 * Return: rc_t
 *  buff          pointer to a pointer to hold a reference to the new Buffer
 *  payload_size  how large to make the data portion of the Buffer
 *  mgr           a reference back to the manager that will control this buffer
 */


/* Standard SRA type constructor:
 * construction of the manager also creates the buffers per specification in the parameters
 *  buffmgr    where to put a reference to the new BufferMgr
 *  buffcount  how many Buffers to make as well
 *  buffsize   how large a payload is desired for the Buffers
 *  timeout    number of milliseconds of wait time for the operations for this BufferMgr
 */


/* run
 *
 * dest will be set if the -o option was used.
 *
 */


/* Init
 *  sets up block
 */


/* Whack
 *  flushes buffer if necessary
 */


/* Write
 *  writes data to buffer, flushes as necessary
 */


/* IndentLevel
 *  increase or decrease indentation level
 */


/* Indent
 *  writes indentation spacing
 */


/* Sep
 *  write separator string
 */


/* Print
 *  \t - indent
 *  \n - end of line
 *  %d - integer
 *  %u - unsigned
 *  %ld - int64_t
 *  %lu - uint64_t
 *  %f - double
 *  %s - null-terminated C-string
 *  %p - separator
 *  %S - String*
 *  %I - unique id
 *  %T - timestamp
 *  %M - MD5 digest
 *  %C - CRC32
 *  %N - CCName*
 *  %F - full CCName*
 */


/* Dump
 *  dump tree using provided callback function
 *
 *  "write" [ IN, NULL OKAY ] and "out" [ IN, OPAQUE ] - callback function
 *  for writing. if "write" is NULL, print to stdout.
 */


/* print root node and call out to print what that contains */


/* filter will let us add the add to and extract by name things to kar */


/* ----------------------------------------------------------------------
 * KSubChunkFileMake
 *  create a new file object
 */


/* ----------------------------------------------------------------------
 * GetSysFile
 *  returns an underlying system file object
 *  and starting offset to contiguous region
 *  suitable for memory mapping, or NULL if
 *  no such file is available.
 *
 * We cant allow memory mapping a tee file as the read?writes ar needed
 * to trigger the writes to the copy KFile
 */


/* ----------------------------------------------------------------------
 * RandomAccess
 *
 *  returns 0 if random access, error code otherwise
 *
 * Update needs to be able to seek both original and copy while read
 * only needs to be able to seek the original.
 */


/* ----------------------------------------------------------------------
 * Type
 *  returns a KFileDesc
 *  not intended to be a content type,
 *  but rather an implementation class
 */


/* ----------------------------------------------------------------------
 * Size
 *  returns size in bytes of file
 *
 *  "size" [ OUT ] - return parameter for file size
 */


/* ----------------------------------------------------------------------
 * SetSize
 *  sets size in bytes of file
 *
 *  "size" [ IN ] - new file size
 */


/* ----------------------------------------------------------------------
 * Read
 *  read file from known position
 *
 *  "pos" [ IN ] - starting position within file
 *
 *  "buffer" [ OUT ] and "bsize" [ IN ] - return buffer for read
 *
 *  "num_read" [ OUT, NULL OKAY ] - optional return parameter
 *  giving number of bytes actually read
 */


/* ----------------------------------------------------------------------
 * Write
 *  write file at known position
 *
 *  "pos" [ IN ] - starting position within file
 *
 *  "buffer" [ IN ] and "size" [ IN ] - data to be written
 *
 *  "num_writ" [ OUT, NULL OKAY ] - optional return parameter
 *  giving number of bytes actually written
 *
 * Unsupported as we now treat archives as READ ONLY
 */


/* -----
 * format is
 * whitechar := {' '|'\f'|'\t'|'\v'}
 * whitespace := whitechar*
 * ext-str = !whitespace!{'\t'|'\n'}*
 * kff-str = !whitespace!{'\t'|'\n'}*
 *
 * A line is
 * [<whitespace>]#<comment line skipped>\n
 * Or
 * [<whitespace>]<ext-str>\t[<whitespace>]<kff-str>\n
 * Or
 * [<whitespace>]\n
 *
 * NOTE: whitespace at the right end of the two strings is included in the strings
 * NOTE: We do not look for '\v'|'\f' within the strings though we maybe should
 * NOTE: the ext string definitely allows white space and punctuation
 */


/* Destroy
 *  destroy FileFormat
 */


/* -----
 * format is
 * whitechar := {' '|'\f'|'\t'|'\v'}
 * whitespace := whitechar*
 * magic-str = !whitespace!{'\t'|'\n'}*
 * kff-str = !whitespace!{'\t'|'\n'}*
 *
 * A line is
 * [<whitespace>]#<comment line skipped>\n
 * Or
 * [<whitespace>]<magic-str>\t[<whitespace>]<kff-str>\n
 * Or
 * [<whitespace>]\n
 *
 * NOTE: whitespace at the right end of the two strings is included in the strings
 * NOTE: We do not look for '\v'|'\f' within the strings though we maybe should
 * NOTE: the magic string definitely allows white space and punctuation
 */


/* Type
 *  returns a KMagicFileFormatDesc
 *  [OUT] rc_t               return
 *  [IN]  const KMagicFileFormat *  self
 *  [IN]  void **            buffer       buffer to hold returned description
 *  [IN]  size_t             buffer_size  size of the buffer
 *  [OUT] char **            descr        text description of file type
 *  [IN]  size_t             descr_max    maximum size of string descr can hold
 *  [OUT] size_t *           descr_len    length of returned descr (not including NUL
 */


/* -----
 * if descr is too short, not NUL terminated, and at the edge of legal memory
 * there is a chance the next two functions could have a segmentation fault
 */


/* Not currently used
static
KFileFormatClass KFFTablesFindKFileFormatClass (KFFTables * self,
						const char * descr )
{
    BSTNode * pbn;
    LOGENTRY (10, "(10, "KFFTablesFindKFileFormatClass"));
    if ((self == NULL) || (descr == NULL))
	return kffcError;
    pbn = BSTreeFind (&self->classtree, descr, classcmp);
    if (pbn == NULL)
	return kffcUnknown;
    return ((KFFClass*)pbn)->class;
}
*/


/* allocate a column-definition */


/*
 * initializes a column-definitions-list
*/


/*
 * destroys the column-definitions-list
*/


/*
 * helper-function for: col_defs_parse_string / col_defs_extract_from_table
 * - creates a column-definition by the column-name
 * - adds the definition to the column-definition-vector
*/


/*
 * calls VTableListReadableColumns to get a list of all column-names of the table
 * creates a temporary read-cursor to test and get the column-type
 * walks the KNamelist with the available column-names
 * tries to add every one of them to the temp. cursor
 * if the column can be added to the cursor, it is appended
 * to the column-definition list
*/


/*
 * walks the list of column-definitions and tries to add all off them
 * to the given cursor, it stops if one of them fails to be added
 * for every column it detects type_decl and type_desc
 * if both calls are successful the column-def is marked as valid
*/


/*
 * retrieves a pointer to a column-definition
 * name defines the name of the wanted column-definition
*/


/*
 * retrieves the index of a column-definition
 * name defines the name of the wanted column-definition
*/


/*
 * retrieves a pointer to a column-definition
 * idx defines the Vector-Index of the wanted column-definition
 * !!! not the read or write cursor index !!!
*/


/*
 * adds all columns marked with to_copy == true to the cursor
*/


/*
 * searches in the column-vector for a successful string-comparison
 * with the hardcoded name of the filter-column
 * stores it's vector-idx in defs->filter_idx
 * does not require an open cursor.
*/


/*
 * walks through the column-names and marks every column thats
 * name is in "redactable_cols" as redactable
 * sets the redact-value to zero
 * if the default-type of the column is in the list of dna-types
 * (a hardcoded list) it sets the redact-value to 'N'
 * does not require an open cursor.
*/


/*
 * walks through the columns and checks every column against
 * a list of types, if one of the src-types of the column
 * is in the list it is marked as redactable
*/


/*
 * walks through the column-definitions and counts how
 * many of them have the to-copy-flag
 * does not require an open cursor.
*/


/*
 * walks through the column-definitions and if a column
 * has the to_copy-flag set, and it's name is in the
 * given list of column-names the to copy-flag is cleared
 * does not require an open cursor.
*/


/* allocate a redact-value */


/*
 * initializes a redact-val-list
*/


/*
 * destroys the redact-val-list
*/


/*
 * adds a entry into the redact-val-list
*/


/*
 * returns a pointer to a redact-value by type-name
*/


/*
 * returns a pointer to a redact-value by type-cast
*/


/*
 * helper-function to set a string inside the context
 * ( makes a copy ) with error detection
*/


/*
 * helper-function to initialize the values of the context
*/


/*
 * generates a new context, initializes values
*/


/*
 * destroys a context, frees all pointers the context owns
*/


/*
 * clear's the number-generator and sets the given intervall
*/


/*
 * helper-function to set the path for additional kfg-files
*/


/*
 * helper-function to set the name of the source-table
*/


/*
 * returns the number of schema's given on the commandline
*/


/*
 * reads all arguments and options, fills the context
 * with copies (if strings) of this data
*/


/* allocate a type-pair */


/* allocate a m-type definition */


/* allocate a matcher-column */


/* initializes the matcher */


/* destroys the matcher */


/* makes a typecast-string for the src/dest-table by column-name */


/* checks if a column with the given name has at least one type
   that is also in the given typelist... */


/* detect the row-range of the source-table
   check if the requested row-range is within this range
   otherwise correct the requested row-range
*/


/* detect the row-range of the source-table
   set's the range of the number-generator to the discovered values
*/


/* find the filter-column and redactable columns */


/* we have the src-columns collected (incl. there types):
   no columns have been added to a cursor (rd and wr),
   now we can find the columns which have to be copied
   including the right type-casts for every one of these columns */


/* this is the common entry-point for the copy-operation in both cases
   "given path is database" OR "given path is table" */


/***************************************************************************
    vdb_copy_perform:
    * called by "vdb_copy_main()"
    * makes the "default-schema"
    * tests if the given source path is a database/table/none-of-that

ctx        [IN] ... contains path, tablename, columns, row-range etc.
vdb_mgr    [IN] ... contains the vdb-manger to create src/dst-objects
***************************************************************************/


/***************************************************************************
    vdb_copy_main:
    * called by "KMain()"
    * make the "native directory"
    * make a vdb-manager for write
      all subsequent copy-functions will use this manager...
    * check if the given path is database-path ( by trying to open it )
      if it is one: continue wit copy_database()
    * check if the given path is table-path ( by trying to open it )
      if it is one: continue wit copy_table()
    * release manager and directory

ctx        [IN] ... contains path, tablename, columns, row-range etc.
***************************************************************************/


/*
 * calls the given manager to create a new SRA-schema
 * takes the list of user-supplied schema's (which can be empty)
 * and let the created schema parse all of them
*/


/*
 * reads a string out of a vdb-table:
 * needs a open cursor, a row-idx and the index of the column
 * it opens the row, requests a pointer via VCursorCellData()
 * malloc's the dst-pointer and copies the string into it
 * a 0-termination get's attached
 * finally it closes the row
*/


/*
 * reads a int64 out of a vdb-table:
 * needs a open cursor, row-id already set, row opened
 * it copies the data via memmove where to the dst-pointer
 * points to / the size in the vdb-table can be smaller
*/


/*
 * reads a int64 out of a vdb-table:
 * needs a open cursor, it opens the row,
 * calls helper_read_vdb_int_row_open()
 * finally it closes the row
*/


/*
 * reads a string out of a KConfig-object:
 * needs a cfg-object, and the full name of the key(node)
 * it opens the node, discovers the size, malloc's the string,
 * reads the string and closes the node
*/


/*
 * reads a uint64 out of a KConfig-object:
 * needs a cfg-object, and the full name of the key(node)
 * it calls helper_read_cfg_str() to read the node as string
 * and converts it via strtou64() into a uint64
*/


/*
 * tries to detect if the given name of a schema-table is
 * a schema-legacy-table, it needs the vdb-manager to
 * open a sra-schema and request this schema to list all
 * legacy tables, then it searches in that list for the
 * given table-name
*/


/*
 * helper-function to replace all ':' and '#' chars
 * with '_' chars
*/


/*
 * helper-function to read the value of cfg-node
 * if key_sel = xxx and s_sub = yyy
 * the node-name is formed like that "/VDBCOPY/xxx/yyy"
 * to read the value helper_read_cfg_str() is called
*/


/*
 * helper-function to read the value of a cfg-node
 * if pf = "_454_", schema = "NCBI_SRA__454__tbl_v0_1"
 * and postfix = "tab" it first tries to read:
 *      "/VDBCOPY/NCBI_SRA__454__tbl_v0_1/tab"
 * if this node is not found it tries to read:
 *      "/VDBCOPY/_454_/tab"
 * if this node is not found it is a error
*/


/*
 * this function tries to read 3 strings out of cfg-nodes
 * (1) the name of the schema-file to be used to copy to
 * (2) the name of the table to be used inside the schema
 * (3) a list of columns that should not be copied
 * if (3) cannot be found it is not a error
 * it needs as input the cfg-object, the name of the
 * (src)legacy-platform and the name of the
 * (src)schema-table
*/


/*
 * this function creates a config-manager, by passing in
 * a KDirectory-object representing the given path
 * this will include the the given path into the search
 * for *.kfg - files
*/


/*
 * this function tries to read the lossynes-score of a vdb-type
 * out of a config-object
 * it constructs a node-name-string like this:
 *      "/VDBCOPY/SCORE/INSDC_2na_bin"
 * INSDC/2na/bin is the name of the type with ":" replaced by "_"
 * it reads the value by calling helper_read_cfg_str()
*/


/* remove the target-table if something went wrong */


/** Init the static directory object */


/** Check file existance */


/** Get next spot id, check whether it should be redacted.
Returns false if maxSpotId reached */


/** The main data structure */


/** Get command line key with argument */


/* because we cannot include <kapp/main.h> where it is defined! */


/***************************************************************************
    perform the actual diffing
***************************************************************************/


/*
 * add the pairs in defs to the given cursor
*/


/*
    - list1 and list2 containts strings
    - if one of the strings in list2 is contained ( partial match, strstr() )
      in one of the strings in list1 the function returns true...
*/


/*  ----------------------------------------------------------------------
 * pull paramstring 1-N and comnvert then to internal VPaths
 */


/*  ----------------------------------------------------------------------
 * SCHEME:PATH/FILE?QUERY
 *
 * form_one is just a file
 * form_two is just a path and a file (can ignore scheme until more schemes supported)
 * form_three is all parts except path present which for here acts like form_one
 * form_four is all four parts
 *
 * path is the directory path leading to root
 * root will be the directory containing the archive
 * base will be the archive as a directory
 */


/*  ----------------------------------------------------------------------
 * KMain
 *
 * Figure out what is on the command line
 */


/* find the first occurence of tag OR if tag doesn't exist, where it should have been. */


/* returns (rcData, rcInsufficient) if eof */


/* MARK: BAM File header parsing functions */


/* MARK: BAM File destructor */


/* file is retained */


/* MARK: BAM Alignment contruction */


/* returns
 *  (rcAlign, rcFile, rcReading, rcBuffer, rcNotAvailable)
 * or
 *  (rcAlign, rcFile, rcReading, rcBuffer, rcInsufficient)
 * if should read with copy
 */


/* MARK: BAM File header info accessor */


/* MARK: BAM Alignment destructor */


/* static bool BAM_AlignmentIsMateMapped(const BAM_Alignment *cself)
{
    if (((getFlags(cself) & BAMFlags_MateIsUnmapped) == 0) && getMateRefSeqId(cself) >= 0 && getMatePos(cself) >= 0)
        return true;
    return false;
} */


/* MARK: end CG stuff */


/* going to make an index with:
 *  a) backing file
 *  b) page size is fixed at 32K, otherwise the page size would have to be stored somewhere
 *  c) page cache - need cache limit in bytes
 *  d) 2 chunked value streams - need key chunking and value chunking
 *  e) B-Tree-ish index, requiring a comparison function. this interface uses opaque keys.
 *     the "matched-count" parameter is useful for trie traversal, but this will be built-in
 */


/* this whacker works fine
 */


/* find or insert will return an existing or create a new entry
 * note that the new entry is initially zeroed
 */


/*TODO: remove when done refactoring */


/* Read
 *  read an aligment
 *
 *  "result" [ OUT ] - return param for BAMAlignment object
 *   must be released with BAMAlignmentRelease
 *
 *  returns RC(..., ..., ..., rcRow, rcNotFound) at end
 */


/* SAM spec says any unrecognized-but-otherwise-valid character gets mapped to N */


/*===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 */


/**
   * Return true if an element has been erased
   */


/**
   * If there is a mapped_type, the mapped value in m_values is not erased now.
   * It will be erased when the ratio between the size of the map and
   * the size of the map + the number of deleted values still stored is low
   * enough (see clear_old_erased_values).
   */


/* special case for SEQUENCE: in order to prepare the cursor correctly, we have to know the first HDF5-source-obj !*/


/* inserts the generated region into the sort_Regions */


/* *************************************************************
if gap is 1, expand previous region by 1 (correct off-by-1)
fill in gaps > 1 ( regions are not consecutive )
correct overlapping regions
fill in a gap at the end, if the last region does not reach spotlen

    INTPUT : rgn->read_Regions
    OUTPUT : rgn->sort_Regions
************************************************************* */


/* *************************************************************
tries to merge overlapping adapter-regions...
( uses rgn->read_Regions as scratch-pad )

    INTPUT : rgn->sort_Regions
    OUTPUT : rgn->sort_Regions
************************************************************* */


/* *************************************************************
tests that the spot-len is covered with regions that
are not overlapping, have no gaps, start is ascending
the regions to check are in rgn->sort_Regions
************************************************************* */


/* read the mapping out of the region-types out of a string... */


/* assembles the 'absolute' path to the requested array-file before opening it */


/* we are reading data from an array-file,
   the underlying array-file knows the size of an element */


/* we are reading values in 2 dimensions from the array-file */


/* was once intended to make the SEQ-table a alias to the CONSENSU-table,
   this step was removed, but the decision could be reinstate again
   because of that the function to do so is still here */


/* we have to pass in the first hdf5-source, because prepare of sequences needs it */


/* KSysDirMake
 *  allocate an uninialized object
 */


/* callback function to be passed into "H5Literate" */


/* translates a flat position into a multi-dim coordinate */


/* not static because imported by hdf5file.c via forward decl. */


/* some usefull utils */
/* strchr but in fixed size buffer (not asciiZ!) */


/* %if-c-only SNIP! this currently causes conflicts with the c++ scanner */
/* yylex_destroy is for both reentrant and non-reentrant scanners. */


/* Return the length of YYSTR.  */


/* make sure all qualities fall into the required range */


/* Handling of multi-line reads and qualities (VDB-3413) */


/* reads 1 line from given file
 * if file->line pointer is not NULL line is in buffer already, nothing is read
 * failes on error or if line empty and not optional
 */


/* reads from a file data for a sinlge spot, data may be partial */


/**
 * SFFLoaderFmtSkipIndex - skip through indexes section if current position in file is indexes offset from header
 */


/* if data is NULL no default value is written, only column is open */


/* BIGGER than internal SRALoaderFile buffer chunks handling
 * skips 'skipover' bytes in buffer and makes 'bsize' bytes available via 'data',
 * caller may want to skip 'skipover' returned back after 'data' is used
 * call with all params zero to deallocated internal buffer
 */


/*
 * assumes tab separated file:
 * first 2 postiions concatinated with "_" into spot prefix
 * nextg 4 postiions concatinated with ":" into spot id: lane:tile:x:y
 * 7th (index) ignored
 * 8th is read id
 * 9th fasta
 * 10th quality
 * 11th (optional) read filter
 */


/*
 * compare seq to quality length ignoring traling spaces, no leading spaces expected
 */


/* parses name as a given word number (1-based) in a str of size len
 * looks for name(#barcode)?([\/.]\d)?
 * returns score of found parts
 * score == 0 word not found
 */


/*
 * in a single line form tries to grab last to chunks defined by sep into seq and qual
 * ignores spaces adjucent to sep
 * normally line would look like "name sep seq sep sep qual"
 */


/*
 * read fasta or quality, which maybe wrapped on 70th column width,
 * into asciiZ buffer
 */


/********** write to toc and archive  */


/********** main create execution  */


/*******************************************************************************
 * Test / Extract
 */


/*
static
void printFile_tree ( const KARFile * file, uint32_t *indent )
{
    KOutMsg ( "%*s%s [ %lu, %lu ]\n", *indent, "", file -> dad . name, file -> byte_offset, file -> byte_size );
}

static
void printDir_tree ( const KARDir *dir, uint32_t *indent )
{
    KOutMsg ( "%*s%s\n", *indent, "", dir -> dad . name );
    *indent += 4;
    BSTreeForEach ( &dir -> contents, false, kar_print, indent );
    *indent -= 4;
}
*/


/*******************************************************************************
 *  Here we will try to resolve path
 */


/* Make
 *  make a standard metadata pair from existing KMetadata objects
 */


/* Release
 *  called by table at end of copy
 */


/* Copy
 *  copy from source to destination
 */


/* IdRange
 *  returns the number of ids available
 *  and optionally the first id
 */


/* Read
 *  read next row of data
 *  returns NULL if no rows are available
 */


/* MakeColumnReader
 *  make simple column reader
 */


/* Release
 *  releases reference
 */


/* MakeColumnWriter
 *  make simple column writer
 */


/* MakeColumnPair
 *  make a simple ColumnPair
 *  takes an optional source VCursor
 *  and a colspec
 */


/* CopyStatic
 *  copy static column from source to destination
 */


/*--------------------------------------------------------------------------
 * StdTblPair
 *  generic database object pair
 */


/* Make
 *  makes an object based upon open source and destination VDatabase objects
 */


/* Explode
 *  probably a bad name, but it is intended to mean
 *  register all enclosed columns
 *  and create a proper pair of KMetadata
 */


/* AddColumnPair
 *  called from implementations
 */


/* Destroy
 *  destroys superclass items
 */


/*--------------------------------------------------------------------------
 * Mem
 *  memory allocation via MemBank in Caps
 */


/* InUse
 *  report the number of bytes in use
 *  optionally returns quota
 */


/* Alloc
 *  allocates some memory from bank
 */


/* Free
 *  returns memory to bank
 *  ignored by paged bank
 *  burden on caller to remember allocation size
 */


/* Make
 *  make a new memory bank
 *  in a real system, this could make a bank linked to
 *  the primordial one. in this system, there is only
 *  a single bank, so this function is overloaded to
 *  create both the primordial and to reset quota on it.
 */


/* Make
 *  creates an id map
 */


/* MakeForPoslen
 *  creates an id map with an additional poslen file
 *  this is specially for PRIMARY_ALIGNMENT_IDS and SECONDARY_ALIGNMENT_IDS
 *  that use *_ALIGNMENT global position and length as a sorting key
 *  we drop it to a file after its generation so that it can be
 *  picked up later when copying the corresponding alignment table.
 */


/* SsetIdRange
 *  required second-stage initialization
 *  must be called before any writes occur
 */


/* SetOldToNew
 *  write old=>new id mappings
 */


/* SetNewToOld
 *  write new=>old id mappings
 */


/* SetPoslen
 *  write global position/length in new-id order
 */


/* ReadPoslen
 *  read global position/length in new-id order
 *  starts reading at the indicated row
 *  returns the number read
 */


/* ReadNewToOld
 *  reads a bunch of new=>old mappings
 */


/* SelectOldToNewPairs
 *  specify the range of new ids to select from
 *  read them in old=>new order
 */


/* SelectOldToNewSingle
 *  specify the range of new ids to select from
 *  read them in old=>new order
 */


/* MapSingleOldToNew
 *  reads a single old=>new mapping
 *  returns new id or 0 if not found
 *  optionally allocates a new id if "insert" is true
 */


/* MakePaged
 *  make a new paged memory bank
 */


/*--------------------------------------------------------------------------
 * StdDbPair
 *  generic database object pair
 */


/* Explode
 *  probably a bad name, but it is intended to mean
 *  register all enclosed tables and databases
 *  and create a proper pair of KMetadata
 */


/* AddDbPair
 *  called from implementations
 */


/* AddTablePair
 *  called from implementations
 */


/* AddDirPair
 *  called from implementations
 */


/* MakePagedMMap
 *  make a new paged memory bank backed by memory map
 */


/* Annotate
 *  make some annotation
 *  but not an error
 */


/* Error
 *  make an annotation
 *  record an error as an rc_t
 */


/* Abort
 *  make an annotation
 *  record an error as an rc_t
 *  exit process
 */


/* ClearError
 *  clears error, leaving any annotation
 */


/* Status
 *  report status
 */


/* POLY_DISPATCH
 *  dispatch a polymorphic message
 */


/* MakeSimpleRowSetIterator
 *  make simple row-id iterator
 *  runs from first to last id in cursor
 */


/*--------------------------------------------------------------------------
 * SortingRowSetIterator
 *  interface to iterate RowSets
 */


/* PreCopy
 *  create buffer
 */


/* PostCopy
 *  douse buffer
 */


/* MakePoslenColReader
 *  make temporary column reader
 */


/*--------------------------------------------------------------------------
 * MappingRowSetIterator
 *  interface to iterate RowSets
 */


/* MakeMappingRowSetIterator
 *  runs from first to last id in cursor
 */


/* MakeBufferedColumnWriter
 *  make a wrapper that buffers all rows written
 *  then sorts on flush
 */


/* MakeBufferedIdRemapColumnWriter
 *  make a wrapper that buffers all rows written
 *  maps all ids through index MapFile
 *  then sorts on flush
 */


/* MakeFirstHalfAlignedRowIdCaptureWriter
 *  a simple monitor on SEQUENCE.PRIMARY_ALIGNMENT_ID looking for
 *  half-aligned spots and capturing the first occurrence.
 */


/*--------------------------------------------------------------------------
 * TestReferenceCell
 *  properly sorted tables will allow both columns to be walked.
 *  each row of the REFERENCE table column MUST contain zero or more
 *  sequential integers, and the first integer MUST continue the sequence
 *  established by previous rows. therefore, the row will be fully specified
 *  by a start and stop id pair.
 */


/*--------------------------------------------------------------------------
 * CrossCheckRefAlignCols
 *  performs the cross-check
 */


/*--------------------------------------------------------------------------
 * CrossCheckRefAlignCurs
 *  adds columns and opens cursors
 */


/*--------------------------------------------------------------------------
 * CrossCheckRefAlignTbl
 *  checks REFERENCE.<name>_IDS for properly sorted form
 *  runs a cross-check of REFERENCE.<name>_IDS against <name>.REF_ID
 */


/* Make
 *  make a standard directory pair from existing KDirectory objects
 */


/* Release
 *  called by db at end of copy
 */


/*--------------------------------------------------------------------------
 * RefTblPair
 *  generic database object pair
 */


/* Init
 *  common initialization code
 */


/* MakeRef
 *  special cased for REFERENCE table
 */


/* MakeAlign
 *  special cased for *_ALIGNMENT tables
 */


/* MakeSeq
 *  special cased for SEQUENCE table
 */


/* Init
 *  initialize a local block from another
 */


/* Whack
 *  release references
 */


/* called by Release */


/* write row to destination */


/* commit all writes */


/* MakeMapRowIdWriter
 *  if "assign_ids" is true, the MapFile will be asked to
 *  map an old=>new id or assign (and record) a new id.
 */


/* INIT
 *  initialize local context block
 */


/* open_db
 *  called from run
 *  determines the type of schema
 *  opens output object
 *  dispatches to the appropriate handler
 */


/* open_tbl
 *  called from run
 *  determines the type of schema
 *  opens output object
 *  dispatches to the appropriate handler
 */


/* Sep. 2022 : to make sure that the column-oriented mode of sra-sort does not
 * invalidate the STATS-metadata-node in all tables of the output database it was
 * requested to unconditionaly copy the STATS-metadata-node of all tables of the
 * src-database into the dst-database
 *
 * This function only succeeds if the dst-database has been closed!
 *
 */


/* run
 *  called from KMain
 *  determines the type of object being copied/sorted
 *  opens input object
 *  dispatches to the appropriate handler
 */


/* MakeRowSetIterator
 *  make RowSet iterator based upon parameters
 *  each RowSet will generate row-ids in src order
 *
 *  "sort_idx" [ IN ]
 *    if NULL, the RowSet objects will auto-generate row-ids
 *    otherwise, src row-ids will be selected from MapFile
 *
 *  "pairs" [ IN ]
 *    if true, the row-ids will be generated into an IdxMapping pair
 *    if false but "idx" is not NULL, the src row-ids will be generated
 *      into a single-id map, and new ordering preserved in parallel
 *    if false and "idx" is NULL, no additional structure will be generated
 */


/**
 *
 * Computes a single row of the kernel matrix, viz. row index.
 *
 */


/**
 *
 * Compute all column sums and the total sum of a kernel matrix.
 *
 */


/**
 *
 * Centers a kernel row as computed by computeKernelRow() using precomputed sums (e.g., sums as given by computeColumnSums(...)).
 *
 */


/* check input arguments */


/* transform upper triangular matrix stored in A as full matrix */


/* Compare function, needed for qsort (needed to sort the times in the HDF5 file) */


/** \todo Implement load() */


/* "View.MemoryView":793
 *
 * @cname('__pyx_memoryview_slice_memviewslice')
 * cdef int slice_memviewslice(             # <<<<<<<<<<<<<<
 *         __Pyx_memviewslice *dst,
 *         Py_ssize_t shape, Py_ssize_t stride, Py_ssize_t suboffset,
 */


/* "View.MemoryView":929
 *
 * @cname('__pyx_memslice_transpose')
 * cdef int transpose_memslice(__Pyx_memviewslice *memslice) nogil except 0:             # <<<<<<<<<<<<<<
 *     cdef int ndim = memslice.memview.view.ndim
 *
 */


/* "View.MemoryView":1192
 *
 * @cname('__pyx_memoryview_copy_data_to_temp')
 * cdef void *copy_data_to_temp(__Pyx_memviewslice *src,             # <<<<<<<<<<<<<<
 *                              __Pyx_memviewslice *tmpslice,
 *                              char order,
 */


/* c translation of the fortran IGM code from Inoue et al. (2014):

http://adsabs.harvard.edu/abs/2014arXiv1402.0677I
http://www.las.osaka-sandai.ac.jp/~inoue/ANAIGM/ANAIGM.tar.gz

*/


/////// Input is chi-squared, convert in place


/////// Input is a probability, not chi-squared


//// Find multiple peaks, distinguished as being above some threshold value, 'thresh'


//// Calculate a "z_m2" like redshift around the peak found in z_peak, but only where p > thresh*pmax


////////// Dump chisq as a function of redshift to [objid].pz file


////////// Dump full best fit template spectrum to [objid].temp_sed file


/////// Dump observed SED file to [objid].obs_sed, including columns of
/////// best-fit templates integrated through the filter curves


//////// Read in zeropoint file


//////// Read in translation file, zphot.translate, and swap columns as defined


//// Read catalog data file


//// Smooth filters if needed and get filter central wavelengths.


//// Read in redshifts computed from an earlier run of the program.
//// Returns (1) if read successfully, (0) if not


///////////////
////  Read wavelength grid
///////////////


///////////////
////  Read template error file
///////////////


//// Read templates and interpolate to wavelength grid


/////// Get IGM shape for full Lyman series, the sawtooth shape of the Madau 95 paper


//////////////
///// Generate template/redshift grid integrated through filters
//////////////


////////////
//// define redshift grid
////////////


//// Initialize z grid, read in filters/data/templates


/// Search through array 'chi2' and find points that are less than
/// their N=2x'Ntest' nearest neighbors.


////// Format of prior file: header line showing mag bins
////// additional lines each with z, p(z|Mag)
//////
////// e.g.
//////  z   18   20    22
////// 0.5  0.9  0.6  0.3
////// 2.5  0.0  0.4  0.5
////// ...
//////
////// each column is therefore the redshift distribution
////// at the magnitude defined in the first line


///////// Interpolate the prior grid to the redshift grid
///////// defined in zphot.param.


//// Multiply p(z) by a wiggle centered at z=1.61 to get rid of z=1.6 NMBS bump


//// Multiply X-band prior to observed p(z) [from chi2] and find where p(z) maximized


///////  "Multiplicative updates for nonnegative quadratic programming ..."
///////    - F. Sha, L. Saul, D. Lee 2006 (earlier version: 2002) [via Google Scholar]
///////
///////
///////   N = N_templates
///////   A = [N x N] matrix where
///////     A[i][j] = sum_over_filters_k(templateI[k]*templateJ[k]/sig2[k])
///////
///////   b = N element vector where
///////     b[i] = -1*sum_over_filters_k(templateI[k]*sourceFlux[k]/sig2[k])
///////
///////   sig2[k] is the squared error on the photometry in filter, k
///////
///////   Difference in Chi^2 between tol=1.e-3 and tol=1.e-7 is only few% and 1e-3 is much faster
///////   Best-fit redshifts are identical at these tolerances (but not at tol=1.e-2).
///////


/****
* Simple linear interpolation
*
* 'istart' speeds up the routine significantly when interpolating
* over a full grid of 'xval's.  Set istart=0 for arbitrary interpolations
* where you're not sure that x[istart] < xval.
*
* The routine as it is assumes that xval < max(x).  Change the 'while' statements
* for more general behavior.
*
*****/


/* "View.MemoryView":765
 *
 * @cname('__pyx_memoryview_slice_memviewslice')
 * cdef int slice_memviewslice(             # <<<<<<<<<<<<<<
 *         __Pyx_memviewslice *dst,
 *         Py_ssize_t shape, Py_ssize_t stride, Py_ssize_t suboffset,
 */


/* "View.MemoryView":901
 *
 * @cname('__pyx_memslice_transpose')
 * cdef int transpose_memslice(__Pyx_memviewslice *memslice) nogil except 0:             # <<<<<<<<<<<<<<
 *     cdef int ndim = memslice.memview.view.ndim
 *
 */


/* "View.MemoryView":1165
 *
 * @cname('__pyx_memoryview_copy_data_to_temp')
 * cdef void *copy_data_to_temp(__Pyx_memviewslice *src,             # <<<<<<<<<<<<<<
 *                              __Pyx_memviewslice *tmpslice,
 *                              char order,
 */


// true if y_max / y_min given as 0.0 in function call


/* Solvable 7 parameter hill model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_core
   1: tau_env
   2: v_lsr
   3: v_in -- infall in the envelope
   4: sigma
   5: To
   6: Tpeak
*/


/* Thin line excitation temperature */


/* Solvable 4 parameter thin line model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_0
   1: v_lsr
   2: sigma
   3: Tex
*/


/* Solvable 6 parameter hill model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_core
   1: tau_env
   2: v_lsr
   3: v_in -- infall in the envelope
   4: sigma
   5: Tpeak
*/


/* relative component intensity */


/* Solvable 5 parameter hill model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_0
   1: v_lsr
   2: v_in
   3: sigma
   4: Tpeak
*/


/* Solvable 4 parameter hyperfine model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_0
   1: v_lsr
   2: sigma
   3: Tex
*/


/* Solvable 6 parameter two layer model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_0
   1: v_lsr
   2: v_in
   3: sigma
   4: Tr
   5: Tf
*/


/*****************************************************************
 ** Reads text file fpin, returns number of data lines, sets retdata to a
 ** two dimensional array of ncol columns and nlines of data. Each line of
 ** the data file must be less than maxlen long. retdata is an initialized
 ** array of pointers to doubles of length ncol. The characters '#' and '!'
 ** as the first character of a line indicate a comment.
 *****************************************************************/


/* Solvable 5 parameter two layer model, calculates fit and returns rms
   residual.

   parameters:
   0: tau_0
   1: v_lsr
   2: v_in
   3: sigma
   4: Tr
*/


/* Solvable 6 parameter hill model with infall in the core, calculates fit
   and returns sumsquare residual.

   parameters:
   0: tau_core
   1: tau_env
   2: v_lsr
   3: v_in -- infall in the core
   4: sigma
   5: Tpeak
*/


/*
   * Add latent heating to HEAT and transfer mass between the phases of each species,
   * as appropriate.
   */


/*========================== terminal_velocity() =================================*/
/*
 * Cs. Palotai
 * Returns terminal velocity [m/s] as a positive number.
 *
 * Inputs:             is: species index
 *                     ip: phase index
 *               pressure: total pressure [Pa]
 *            temperature: physical temperature [K]
 *         precip_density: model precipitation density [kg/m^3]
 */


/*
 * Allocate memory for data tables and read in values.
 */


/*
 * Generic enthalpy change calculation between two states of the given species.
 * C.J.Palotai *A* 06.2003.
 *
 * The enthalpy data tables are in kJ/kg, which is converted here to J/kg.
 */


/*
 * Returns the enthalpy change of water vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of ammonia vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of hydrogen sulfide vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of methane vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of acetylene vapor between two temperatures, in J/kg.
 * NOTE: Need to add data.
 */


/*
 * Returns the enthalpy change of ethylene vapor between two temperatures, in J/kg.
 * NOTE: Need to add data.
 */


/*
 * Returns the enthalpy change of ethane vapor between two temperatures, in J/kg.
 * NOTE: Need to add data.
 */


/*
 * Returns the enthalpy change of carbon dioxide vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of ammonium hydrosulfide vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of ozone vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of nitrogen vapor between two temperatures, in J/kg.
 * C.J.Palotai *A* 03.2003.
 */


/*
 * Returns the enthalpy change of phosphine vapor between two temperatures, in J/kg.
 * Need data for this function.
 */


/*
 * From Table 1.20 of Lodders and Fegley, 1998,
 * The Planetary Scientist's Companion, Oxford University Press.
 * The parameter a has been modified to yield pressure in Pa instead of bar.
 * Temperature in Kelvin.
 *
 * 06/2003 CJP *A*
 * Added case for mixed phase: Supercooled liquid data taken from:
 * Rogers&Yau: A Short Course in Cloud Physics,3rd Ed., p16.
 */


/*
 * From Table 1.20 of Lodders and Fegley, 1998,
 * The Planetary Scientist's Companion, Oxford University Press.
 * The parameter a has been modified to yield pressure in Pa instead of bar.
 * Temperature in Kelvin.
 */


/*
 * NOTE: Need this function.
 */


/*
 * Need data for this function.
 */


/*
 * Code for calculating dynamic viscosity
 *           CJP *A*   4/12/2004
 *        Jupiter: C.F. Hansen (1979)
 */


/*
 * Code for calculating thermal coductivity
 *           CJP *A*   4/12/2004
 *        Jupiter: C.F. Hansen (1979)
 */