/*
 * @class hsaRfiZeroData
 * @brief hsaCommand to zero input data marked as containing large amounts of RFI
 *
 * This is an hsaCommand that launches the kernel (rfi_chime_zero.hsaco) to set all
 * values detected to have a significant amount of RFI to zero
 *
 * @requires_kernel    rfi_chime_zero.hasco
 *
 *
 * @par GPU Memory
 * @gpu_mem  input              Input data of size input_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c uint8_t
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  mask               The RFI mask to be aplied
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c uint8_t
 *     @gpu_mem_metadata        chimeMetadata
 *
 * @conf   num_elements         Int. Number of elements.
 * @conf   num_local_freq       Int. Number of local freq.
 * @conf   samples_per_data_set Int. Number of time samples in a data set.
 * @conf   sk_step              Int (default 256). Length of time integration in SK estimate.
 *
 * @author Jacob Taylor
 */


/*
 * @class hsaRfiBadInput
 * @brief hsaCommand to average single input kurtosis values across a packet.
 *
 * This is an hsaCommand that launches the kernel (rfi_bad_input.hsaco) to compute and average
 * spectral kurtosis estimates from individual inputs. These values are then used to determine
 * which inputs are not functioning properly.
 *
 * @requires_kernel    rfi_bad_input.hasco
 *
 * @par GPU Memory
 * @gpu_mem  input              Input data of size input_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c uint8_t
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  output             Output data of size output_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  M                  The time integration length (samples)
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Constant @c uint32_t
 *     @gpu_mem_metadata        none
 * @gpu_mem  num_sk             The total number of SK estimates to be averaged
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Constant @c uint32_t
 *     @gpu_mem_metadata        none
 *
 * @conf   num_elements         Int. Number of elements.
 * @conf   num_local_freq       Int. Number of local freq.
 * @conf   samples_per_data_set Int. Number of time samples in a data set.
 * @conf   sk_step              Int (default 256). Length of time integration in SK estimate.
 *
 * @author Jacob Taylor
 */


/*
 * @class hsaRfiOutput
 * @brief hsaCommand for copying RFI output gpu to host.
 *
 * This is an hsaCommand that async copy RFI output buffer from GPU
 * to CPU. It marks the RFI output buffer to be full when done so that
 * it can be reused. This code also passes metadata along. The finalize_frame
 * function has been hacked by not marking _network_buf empty, for
 * concurrent run with N2, because the equivalent output code from the N2
 * side is already marking network buffer empty.
 *
 * @par GPU Memory
 * @gpu_mem  rfi_output      Output of SK estimates, size: sizeof_float x nfreqs x nsamps / sk_step
 *     @gpu_mem_type         staging
 *     @gpu_mem_format       Array of @c float
 *     @gpu_mem_metadata     chimeMetadata
 *
 * @author Jacob Taylor
 */


/*
 * @class hsaRfiMaskOutput
 * @brief hsaCommand for copying RFI mask from gpu to host.
 *
 * This is an hsaCommand that async copies the RFI Mask buffer from GPU
 * to CPU. It marks the RFI Mask buffer to be full when done so that
 * it can be reused. This code also passes metadata along.
 *
 * Note: This commands MUST only be used before the hsaOutputData command. This command presumes
 * that the network buffer has not been marked empty yet.
 *
 * @par GPU Memory
 * @gpu_mem  rfi_mask_output The RFI mask, array of uint_t, 0 means not masked, 1 means masked
 *     @gpu_mem_type         staging
 *     @gpu_mem_format       Array of @c float
 *     @gpu_mem_metadata     chimeMetadata
 *
 * @author Jacob Taylor
 */


/*
 * @class hsaRfiVarOutput
 * @brief hsaCommand for copying RFI variance output gpu to host.
 *
 * This is an hsaCommand that async copy RFI variance output buffer from GPU
 * to CPU. It marks the RFI output buffer to be full when done so that
 * it can be reused. This code also passes metadata along. The finalize_frame
 * function has been hacked by not marking _network_buf empty, for
 * concurrent run with N2, because the equivalent output code from the N2
 * side is already marking network buffer empty.
 *
 * @par GPU Memory
 * @gpu_mem  rfi_output_var  Output of SK variances estimates, size: sizeof_float x nfreqs x nsamps
 * / sk_step
 *     @gpu_mem_type         staging
 *     @gpu_mem_format       Array of @c float
 *     @gpu_mem_metadata     chimeMetadata
 *
 * @author James Willis
 */


/*
 * @class hsaRfiTimeSum
 * @brief hsaCommand to performs prallel sum of power and square power across time.
 *
 * This is an hsaCommand that launches the kernel (rfi_chime_time_sum.hsaco) to perform
 * a parallel sum of power and square power estimates across time. The sum is then normalized
 * by the mean power and sent to the hsaRfiIputSum command.
 *
 * @requires_kernel    rfi_chime_time_sum_private.hasco
 *
 * @par REST Endpoints
 * @endpoint    /rfi_time_sum_callback/<gpu_id> ``POST`` Change kernel parameters
 *              requires json values      "bad_inputs"
 *              update config             "bad_inputs"
 *
 * @par GPU Memory
 * @gpu_mem  input              Input data of size input_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c uint8_t
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  time_sum           Output data of size output_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 *
 * @conf   num_elements         Int. Number of elements.
 * @conf   num_local_freq       Int. Number of local freq.
 * @conf   samples_per_data_set Int. Number of time samples in a data set.
 * @conf   sk_step              Int (default 256). Length of time integration in SK estimate.
 * @conf   bad_inputs           Array of Int The inputs which are currently malfunctioning
 *
 * @author Jacob Taylor
 */


/**
 * @class hsaBeamformUpchan
 * @brief hsaCommand to upchannelize and downsample FRB data
 *
 * This is an hsaCommand that launches the kernel (upchannelize_flip) for
 * upchannelizing and downsampling FRB data. First, 128 time samples
 * is traded for 128 upchannelized freq channels via FFT. The freq axis is
 * "flipped" (or more accurately, rolled) so that the higher power is in
 * the middle of the bandpass. Then, every 8 freqs and 3 time samples are
 * averaged in order to downsample to fit the output bandwidth. The 2
 * polarizations are summed to produce a final output in float of
 * 1024 beams x 128 times x 16 freq.
 *
 * @requires_kernel    upchannelize_flip.hasco
 *
 * @par GPU Memory
 * @gpu_mem  transposed_output  Input data of size input_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  bf_output          Output data of size output_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 *
 * @conf   num_elements         Int (default 2048). Number of elements
 * @conf   samples_per_data_set Int (default 49152). Number of time samples in a data set
 * @conf   downsample_time      Int (default 3). Downsample factor in time
 * @conf   downsample_freq      Int (default 8). Downsample factor in freq
 * @conf   num_frb_total_beams  Int (default 1024). Number of total FRB formed beams
 *
 * @author Cherry Ng
 *
 */


/**
 * @class hsaBeamformReorder
 * @brief hsaCommand to reorder input for FRB beamform
 *
 * This is an hsaCommand that launches the kernel (reorder.hsaco) for
 * reordering input data from correlator order to cylinder
 * order with polarization index last (A0-B0-C0-D0-A1-B1-C1-D1)
 * which is convenient for FFT beamforming for the
 * downstream FRB pipeline.
 * An array of reordering index of length 512 is parsed from
 * the config file. This is sufficient to reorder the 2048
 * input elements because the inputs are `scrambled' in groups of 4,
 * i.e., every 4 consecutive inputs are still consecutive in the
 * scrambled order. See doclib #546 for details.
 *
 * @requires_kernel    reorder.hsaco
 *
 * @par GPU Memory
 * @gpu_mem input            Input data of size input_frame_len
 * @gpu_mem_type             staging
 * @gpu_mem_format           Array of @c uchar
 * @gpu_mem_metadata         chimeMetadata
 *
 * @gpu_mem reorder_map      Array of reordering index of size 512
 * @gpu_mem_type             static
 * @gpu_mem_format           Array of @c uint8
 * @gpu_mem_metadata         none
 *
 * @gpu_mem input_reordered  Reordered data
 * @gpu_mem_type             static
 * @gpu_mem_format           Array of @c uchar
 * @gpu_mem_metadata         chimeMetadata
 *
 * @conf  num_elements          Int (default 2048). Number of elements
 * @conf  samples_per_data_set  Int (default 49152). Number of time samples in a data set
 * @conf  reorder_map           Int (array of default size 512). Reordering index
 * @conf  num_local_freq        Int (default 1). Number of local frequencies.
 *
 * @todo  Change output name to something more sensible, silly to have "input" as output.
 *
 * @author Cherry Ng
 *
 */


/**
 * @class hsaBeamformUpchanHFB
 * @brief hsaCommand to upchannelize and downsample FRB data
 *
 * This is an hsaCommand that launches the kernel (upchannelize_flip) for
 * upchannelizing and downsampling FRB data. First, 128 time samples
 * is traded for 128 upchannelized freq channels via FFT. The freq axis is
 * "flipped" (or more accurately, rolled) so that the higher power is in
 * the middle of the bandpass. Then, every 8 freqs and 3 time samples are
 * averaged in order to downsample to fit the output bandwidth. The 2
 * polarizations are summed to produce a final output in float of
 * 1024 beams x 128 times x 16 freq.
 *
 * Hyper fine beam data is also output at this stage. Before the FRB data is
 * downsampled, both polarizations are summed over all time samples for every
 * frequency. A bandpass filter is applied and the final output is in float of
 * 1024 beams x 128 freqs x 10 samples
 *
 * @requires_kernel    upchannelize_flip.hasco
 *
 * @par GPU Memory
 * @gpu_mem  transposed_output  Input data of size input_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  bf_output          Output data of size output_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  hfb_output         Output data of size output_hfb_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c float
 *
 * @conf   num_elements         Int (default 2048). Number of elements
 * @conf   samples_per_data_set Int (default 49152). Number of time samples in a data set
 * @conf   downsample_time      Int (default 3). Downsample factor in time
 * @conf   downsample_freq      Int (default 8). Downsample factor in freq
 * @conf   num_frb_total_beams  Int (default 1024). Number of total FRB formed beams
 * @conf   factor_upchan        Int (default 128). Upchannelise factor
 *
 * @author Cherry Ng and James Willis
 *
 */


/*
 * @class hsaRfiBadInputOutput
 * @brief hsaCommand for copying RFI bad input kernel output gpu to host.
 *
 * This is an hsaCommand that async copy RFI bad input kernel output buffer from GPU
 * to CPU. It marks the output buffer to be full when done so that it can be reused.
 * This code also passes metadata along.
 *
 * @par GPU Memory
 * @gpu_mem  rfi_bad_input   Output of averaged SK estimates for individual inputs.
 *     @gpu_mem_type         staging
 *     @gpu_mem_format       Array of @c float
 *     @gpu_mem_metadata     chimeMetadata
 *
 * @author Jacob Taylor
 */


/**
 * @class hsaBeamformHFBOutput
 * @brief hsaCommand for copying HFB output gpu to host.
 *
 * This is an hsaCommand that async copy HFB output buffer from GPU
 * to CPU. It marks the HFB output buffer to be full when done so that
 * it can be reused.
 *
 * @par GPU Memory
 * @gpu_mem  hfb_output      Output from the HFB pipeline, size 1024x128x4 (beams x sub-freq x
 * sizeof(float))
 *     @gpu_mem_type         staging
 *     @gpu_mem_format       Array of @c float
 *
 * @author James Willis
 *
 */


/*
 * @class hsaHostToDeviceCopy
 * @brief hsaCommand for transferring a CPU buffer to GPU staging buffer
 *
 * @par buffers
 * @buffer in_buf The host buffer to transfer data to the GPU from
 *      @buffer_format any
 *      @buffer_metadata any
 *
 * @config gpu_memory_name    String, the name of GPU side frames
 *
 * @par GPU Memory
 * @gpu_mem  Data from host side buffer after copy with name @c gpu_memory_name
 *     @gpu_mem_type         staging
 *
 * @author Andre Renard
 */


/**
 * @class hsaBeamformKernel
 * @brief hsaCommand to beamform for FRB
 *
 * This is an hsaCommand that launches the kernel (unpack_shift_beamform_flip) for
 * FRB beamforming. The kernel unpacks the already reordered input data (shape @c
 * n_samp x @c n_elem) and multiplies the conjugate of gains to it. We then apply
 * an FFT beamforming along the N-S direction that is padded by 2. An array of
 * clamping index is used to tell which of the 512 beams to clamp to, to form 256
 * N-S beams. A brute force beamform along the E-W direction is calculated using an
 * array of phase delays to form 4 E-W beams. The N-S beam extent and the 4 E-W beam
 * positions are both tunable via endpoints. The output is flipped in the N-S
 * direction in order to following the L2/3 convention of south beams before north
 * beams. The ordering of the output data is time-pol-beamEW-beamNS, where beamNS
 * is the fastest varying.
 *
 * The gain path is registered as a subscriber to an updatable config block.
 *
 * @requires_kernel    unpack_shift_beamform_flip.hasco
 *
 * @par REST Endpoints
 * @endpoint    /frb/update_NS_beam/\<gpu id\> ``POST`` Trigger re-set of
 *              FFT beam spacing in N-S
 *              requires json values      northmost_beam
 *              update config             northmost_beam
 * @endpoint    /frb/update_EW_beam/\<gpu id\> ``POST`` Trigger re-calculate
 *              of phase delay for the 4 E-W brute-force formed beams
 *              requires json values      ew_id, ew_beam
 *              update config             ew_spacing[ew_id]
 *
 * @par GPU Memory
 * @gpu_mem  input_reordered    Input data of size input_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c uchar
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  beamform_output    Output data of size output_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  beamform_map       Array of clamping index of size 256
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c uint
 *     @gpu_mem_metadata        none
 * @gpu_mem  beamform_coeff_map Array of phase delay size 32
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        none
 * @gpu_mem  beamform_gain      Array of gains size 2048*2
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        none
 *
 * @conf   num_elements         Int (default 2048). Number of elements
 * @conf   num_local_freq       Int (default 1). Number of local freq.
 * @conf   samples_per_data_set Int (default 49152). Number of time samples in a data set
 * @conf   scaling              Float (default 1.0). Scaling factor on gains
 * @conf   default_gains        Float array (default 1+1j). Default gain value if gain file is
 * missing
 * @conf   northmost_beam       Float - Setting the extent in NS of the FFT formed beams.
 *                              Zenith angle of the northmost beam (in deg).
 * @conf   ew_spacing           Float array - 4 sky angles for the columns of E-W beams (in deg).
 *
 * @todo   Better handle of variables that gets updated via endpoint, prevent
 *         read/write conflicts.
 *
 * @author Cherry Ng
 *
 */


/**
 * @class hsaBeamformTranspose
 * @brief hsaCommand to transpose FRB data from time-pol-beam to pol-beam-time
 *
 * This is an hsaCommand that transposes FRB output from the beamform
 * kernel with format time-pol-beam to pol-beam-time, in order to
 * minimize striding for the next step of upchannelization, which
 * needs time to be fastest varying. 32 blank columns are padded to the output
 * of this transpose for GPU memory access optimization.
 *
 * @requires_kernel    transpose.hasco
 *
 * @par GPU Memory
 * @gpu_mem  beamform_output Input data of size beamform_frame_len
 *     @gpu_mem_type         static
 *     @gpu_mem_format       Array of @c float
 *     @gpu_mem_metadata     chimeMetadata
 * @gpu_mem  bf_output       Output data of size output_frame_len
 *     @gpu_mem_type         static
 *     @gpu_mem_format       Array of @c float
 *     @gpu_mem_metadata     chimeMetadata
 *
 * @conf   num_elements         Int (default 2048). Number of elements
 * @conf   samples_per_data_set Int (default 49152). Number of time samples in a data set
 *
 * @author Cherry Ng
 *
 */


/**
 * @class hsaRfiUpdateBadInputs
 * @brief hsaCommand copy-in command which updates the bad input list for the RFI system
 *
 * @par Buffers:
 * @buffer in_buf Buffer containing the list of bad inputs
 *     @buffer_format Array of @c uint8_t
 *     @buffer_metadata chime_metadata

 * @buffer network_buf Buffer containing the metadata for
 *     @buffer_format FPGA post PDF data.
 *     @buffer_metadata chime_metadata
 *
 * @author Andre Renard & James Willis
 */


/**
 * @class hsaBeamformHFBSum
 * @brief hsaCommand to upchannelize and downsample FRB data
 *
 * This is an hsaCommand that launches the kernel (sum_hfb) for
 * summing HFB data across all samples. Excludes samples from
 * the sum where there is >=1 lost samples. Final output in float of
 * 1024 beams x 128 freq.
 *
 * @requires_kernel   sum_hfb.hasco
 *
 * @par GPU Memory
 * @gpu_mem  hfb_output  Input data of size input_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 * @gpu_mem  compressed_lost_samples Array indicating if a given timestep was zeroed, size:
 *           samples_per_data_set / factor_upchan
 *     @gpu_mem_type         staging
 *     @gpu_mem_format       Array of @c uint8_t
 *     @gpu_mem_metadata     chimeMetadata
 * @gpu_mem  hfb_sum_output     Output data of size output_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c float
 *
 * @conf   num_frb_total_beams  Int (default 1024). Number of total FRB formed beams
 * @conf   factor_upchan  Int (default 128). Upchannelise factor
 * @conf   samples_per_data_set Int. Number of time samples in a data set.
 * @conf   num_samples Int. Number of samples per HFB frame.
 *
 * @author James Willis
 *
 */


/*
 * @class hsaRfiInputSum
 * @brief hsaCommand to compute the input sum and spectral kurtosis for RFI detection
 *
 * This is an hsaCommand that launches the kernel (rfi_chime_input_sum.hsaco) to perform
 * a sum of normalized, time summed square power estimates (see hsaRfiTimeSum.hpp). The
 * sum is then used to calculate a spectral kurtosis estimate. The spectral kurtosis estimate
 * is a measure of the underlying gaussianity of the sample. Thus it can be used as a tool
 * to detect non-gaussian signals in CHIME's incoherent beam (RFI).
 *
 * @requires_kernel    rfi_chime_input_sum.hasco
 *
 * @par REST Endpoints
 * @endpoint    /rfi_input_sum_callback/<gpu_id> ``POST`` Change kernel parameters
 *              requires json values      "num_bad_inputs"
 *              update config             "num_bad_inputs"
 *
 * @par GPU Memory
 * @gpu_mem  time_sum           Input data from the hsaRfiTimeSum command  of size input_frame_len
 *     @gpu_mem_type            static
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  rfi_output          Output data of size output_frame_len
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  input_mask         Inputs to be ignored/masked (1 for mask, 0 for don't mask)
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c uint8_t
 *     @gpu_mem_metadata        chimeMetadata
 * @gpu_mem  rfi_mask_output    Mask used to zero input data (1 for RFI, 0 for clean)
 *     @gpu_mem_type            staging
 *     @gpu_mem_format          Array of @c float
 *     @gpu_mem_metadata        chimeMetadata
 *
 * @conf   num_elements         Int. Number of elements.
 * @conf   num_local_freq       Int. Number of local freq.
 * @conf   samples_per_data_set Int. Number of time samples in a data set.
 * @conf   sk_step              Int (default 256). Length of time integration in SK estimate.
 * @conf   bad_inputs           Array of Ints. Used to compute the number of faulty inputs
 *
 * @author Jacob Taylor
 */


/**
 * @class applyGains
 * @brief Receives gains and apply them to the output buffer.
 *
 * This stage registers as a subscriber to an updatable config block. The
 * full name of the block should be defined in the value @c updatable_block
 *
 * Gain updates *must* match the frequencies expected to be present in the
 * input stream. That is there must be exactly as many frequencies in the gain
 * update as there are in the `freqState` attached to the input stream.
 * The number of elements must also match those on the incoming
 * stream.
 *
 * The number of frequencies and inputs is locked in
 *
 *
 * @par Buffers
 * @buffer in_buf The input stream.
 *         @buffer_format VisBuffer.
 *         @buffer_metadata VisMetadata
 * @buffer out_buf The output stream.
 *         @buffer_format VisBuffer.
 *         @buffer_metadata VisMetadata
 *
 * @conf   num_elements     Int.    The number of elements (i.e. inputs) in the
 *                                  correlator data.
 * @conf   updatable_block  String. The full name of the updatable_block that
 *                                  will provide new flagging values (e.g. "/dynamic_block/gains").
 * @conf   gains_dir        String. The path to the directory holding the gains file.
 * @conf   broker_host      String. Calibration broker host.
 * @conf   broker_port      Int.    Calibration broker port.
 * @conf   read_from_file   Bool, default false. Whether to read the gains from file or
 *                                  fetch them over the network.
 * @conf   tcombine         Double. Time (in seconds) over which to combine old and new gains to
 *                                  prevent discontinuities. Default is 5 minutes.
 * @conf   num_kept_updates Int.    The number of gain updates stored in a FIFO.
 * @conf   num_threads      Int.    Number of threads to run. Default is 1.
 *
 * @par Metrics
 * @metric kotekan_applygains_late_update_count The number of updates received
 *     too late (The start time of the update is older than the currently
 *     processed frame).
 * @metric kotekan_applygains_late_frame_count The number of frames received
 *     late (The frames timestamp is older then all start times of stored
 *     updates).
 * @metric kotekan_applygains_update_age_seconds The time difference in
 *     seconds between the current frame being processed and the time stamp of
 *     the gains update being applied.
 *
 * @author Mateus Fandino, Tristan Pinsonneault-Marotte and Richard Shaw
 */


/**
 * @class rawFileWrite
 * @brief Stream a buffer to disk.
 *
 * @par Buffers:
 * @buffer in_buf Buffer to write to disk.
 *     @buffer_format Any
 *     @buffer_metadata Any
 *
 * @conf base_dir  String. Directory to write into.
 * @conf file_name String. Base filename to write.
 * @conf file_ext  String. File extension.
 * @conf num_frames_per_file Int. No of frames to write into a single file.
 * @conf exit_after_n_files  Int. Stop writing after this many files, Default 0 = unlimited files.
 *
 * @par Metrics
 * @metric kotekan_rawfilewrite_write_time_seconds
 *         The write time to write out the last frame.
 *
 * @author Andre Renard
 **/


/**
 * @class monitorBuffer
 * @brief Watches a set of buffers to make sure they are getting data
 *        within a given timeout, otherwise exits.
 *
 * @note This will not detect buffers which never get data, the buffer
 *       must get at least one frame before this system will start checking.
 *
 * @par Buffers
 * @buffer bufs An array of kotekan buffers
 *  @buffer_format Any
 *  @buffer_metadata Any
 *
 * @conf timeout       Int, default 60. Time to wait in seconds for new frame
 *                     before exiting.   Must be > 1
 * @conf fill_threshold  Float, default 2.0 (disabled)  The ratio of full to total frames,
 *                       which if exceeded with trigger an exit.
 */


/**
 * @brief Splits the input buffer into multiple output frames in a round robin fashion.
 *
 * Uses zero copy operations if this stage is the only consumer, if not it uses a deep copy.
 * This stage must be the only producer of the output buffers
 *
 * @par Buffers
 * @buffer in_buf The source buffer
 *        @buffer_format Matches the output buffers
 *        @buffer_metadata Matches the output buffers
 *
 * @buffer out_bufs Array of buffers to fill with frames from in_buf
 *        @buffer_format any, but all must be the same type and match in_buf
 *        @buffer_metadata any, but all must be the same type match in_buf
 *
 * @author Andre Renard
 */


/**
 * @class prodSubset
 * @brief ``kotekan::Stage`` that extracts a subset of the products.
 *
 * This task consumes a full set of visibilities from a ``VisBuffer`` and
 * passes on a subset of products to an output ``VisBuffer``. The subset
 * extracted depends on the parameter 'prod_subset_type'. Here is a list of
 * values 'prod_subset_type' can take and the parameters they support:
 * - 'all': no extra arameters needed
 *   - Default. All the products.
 * - 'autos': no extra arameters needed
 *   - The subset are all the auto-correlations.
 * - 'baseline': max_ew_baseline, max_ns_baseline
 *   - Selects a subset of products whose baseline length is smaller than the
 *   maximum values given by config parameters in the EW and NS directions.
 * - 'have_inputs': input_list
 *   - The subset are all the correlations containing at leat one of the
 *   inputs in the input_list.
 * - 'only_inputs': input_list
 *   - The subset are all the correlations containing only inputs from the
 *   input_list.
 *
 * @par Buffers
 * @buffer in_buf The kotekan buffer from which the visibilities are read, can be any size.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 * @buffer out_buf The kotekan buffer which will be fed the subset of visibilities.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 *
 * @conf  prod_subset_type      string. Type of product subset to perform.
 * @conf  num_elements          int. The number of elements (i.e. inputs) in the
 *                              correlator data
 * @conf  num_ev                int. The number of eigenvectors to be stored
 * @conf  max_ew_baseline       int. The maximum baseline length along the EW
 *                              direction to include in subset (in units of the
 *                              shortest EW baseline)
 * @conf  max_ns_baseline       int. The maximum baseline length along the NS
 *                              direction to include in subset (in units of the
 *                              shortest NS baseline)
 * @conf  input_list            vector of int. The list of inputs to include.
 *
 *
 * @warning This will only work correctly if the full correlation triangle is
 * passed in as input.
 *
 * @author  Tristan Pinsonneault-Marotte and Mateus Fandino
 *
 */


/**
 * @class VisTruncate
 * @brief Truncates visibility, eigenvalue and weight values.
 *
 * eigenvalues and weights are truncated with a fixed precision that is set in
 * the config. visibility values are truncated to a precision based on their
 * weight.
 *
 * @warning Don't run this anywhere but on the transpose (gossec) node.
 * The OpenMP calls could cause issues on systems using kotekan pin
 * priority threads (likely the GPU nodes).
 *
 * @par Buffers
 * @buffer in_buf The input stream.
 *         @buffer_format VisBuffer.
 *         @buffer_metadata VisMetadata
 * @buffer out_buf The output stream with truncated values.
 *         @buffer_format VisBuffer.
 *         @buffer_metadata VisMetadata
 *
 * @conf   err_sq_lim               Limit for the error of visibility truncation.
 * @conf   weight_fixed_precision   Fixed precision for weight truncation.
 * @conf   data_fixed_precision     Fixed precision for eigenvector and visibility truncation (if
 * weights are zero).
 *
 * @author Tristan Pinsonneault-Marotte, Rick Nitsche
 */


/**
 * @class bufferStatus
 *
 * @brief Exports buffer metrics and prints out buffer status
 *
 * Exports buffer size and current load (number of full buffers)
 * for all buffers in the system, and prints this information
 * to the logs and/or stdout depending on the system settings.
 *
 * Note at the moment this class requires the global log_level to be
 * set to INFO or highter for the buffer metrics to be output to the logs.
 *
 * @conf time_delay    Int. Microseconds, default 1000000 (1 second)
 *                       The number of micro seconds between buffer print outs.
 *                       Will not output more frequently than 100ms.
 * @conf print_status  Bool. Default true.
 *                       If true buffer stats are send to the logs/stderr
 *
 * @par Metrics
 * @metric kotekan_bufferstatus_full_frames_total
 *         The number of full frames for a given buffer
 * @metric kotekan_bufferstatus_frames_total
 *         The total number of frames in a given buffer (buffer depth)
 *
 * @author Jacob Taylor, Andre Renard
 */


/**
 * @class visTestPattern
 * @brief Checks if the visibility data matches a given expected pattern (CHIME specific).
 *
 * This stage is just dropping incoming frames until it receives an update on what data to expect.
 * An update would be sent to the endpoint `/run-test`.  *
 * This stage will run the test for the given number of frames, then send a request to a
 * reply-endpoint to signal that the test is done. The reply endpoint is expected in the update.
 * In case there was an error during the test, it replies
 * with the error message in the result field.
 * Afterwards it will stay idle until the next test is started.
 *
 * Errors are calculated as the norm of the difference between the expected and
 * the actual (complex) visibility value.
 * For bad frames, the following data is written to a csv file specified in the
 * config:
 * fpga_count:  FPGA counter for the frame
 * time:        the frames timestamp
 * freq_id:     the frames frequency ID
 * num_bad:     number of values that have an error higher then the threshold
 * avg_err:     average error of bad values
 * min_err:     minimum error of bad values
 * max_err:     maximum error of bad balues
 *
 * Additionally a report is printed to the log in a configured interval.
 *
 * @par REST Endpoints
 * @endpoint    /run-test ``POST``  Updates the FPGA test pattern to expect and how long to run a
 *                                  test on that pattern. Required json values:
 *              name            String. Name of the test.
 *              num_frames      Int. Number of frames to check for the given test pattern. If this
 *                              is `0`, this stage will stay idle until a new update is received.
 *              test_pattern    Dictionary of String -> (List of uint8). Mapping of correlator
 *                              input serial number to a list of values for each frequency bin.
 *                              For example: `{'FCCXXYYZZ': [2048 list of uint8], ...}`, where the
 *                              2048 uint8 are the real and imaginary component for each frequency
 *                              bin and FCCXXYYZZ is the serial number of the correlator input.
 *              reply_host      String. Hostname to send test-done message to.
 *              reply_path      String. Endpoint path to send test-done message to.
 *              reply_port      Int. Port to send test-done message to.
 *
 * @par Buffers
 * @buffer in_buf               The buffer to debug
 *         @buffer_format       VisBuffer structured
 *         @buffer_metadata     VisMetadata
 * @buffer out_buf              All frames found to contain errors
 *         @buffer_format       VisBuffer structured
 *         @buffer_metadata     VisMetadata
 *
 * @conf  write_dir             String. Path to the place to dump all output in.
 * @conf  report_freq           Int. Number of frames to print a summary for (default: 1000).
 * @conf  tolerance             Float. Defines what difference to the expected value is an error
 *                              (default: 1e-6).
 * @conf  endpoint_name         String. Name for the update endpoint (default: "run_test").
 *
 * @author Rick Nitsche
 */


/**
 * @class testDataGen
 * @brief Generate test data as a standin for DPDK.
 *
 * @par Buffers
 * @buffer out_buf Buffer to fill
 *         @buffer_format any format
 *         @buffer_metadata chimeMetadata
 *
 * @conf  type                  String. "const", "random", "random_signed", "ramp", or "tpluse".
 * @conf  value                 Int. Required for type "const" and "ramp".
 * @conf  values                Vector of ints. Only used for type "onehot" - sets the array element
 * to a different value for each frame; loops through the values.
 * @conf  seed                  Int. For type "random", "random_signed", and "onehot".  If non-zero,
 * seeds the random number generator on startup for reproducible results.
 * @conf  reuse_random          Bool, default False.  For "random" types, only generate each random
 * block once, then keep re-using it.
 * @conf  wait                  Bool, default True. Produce data a set cadence.
 *                              Otherwise just as fast as possible.
 * @conf  samples_per_data_set  Int. How often to produce data.
 * @conf  stream_id             Int.
 * @conf  num_frames            Int. How many frames to produce. Default inf.
 * @conf  num_freq_in_frame     Int. Number of frequencies in each GPU frame.
 * @conf  rest_mode             String. "none" (default), "start", or "step.
 *                              How to interact with rest commands to trigger
 *                              data production.
 * @conf  num_links             Int.  How many links are being simulated, impacts
 *                              the rate of data generated in the wait = true mode.
 *
 * @par REST Endpoints
 * @endpoint /\<unique_name\>/generate_test_data
 *             ``POST`` Triggers the generation of data.
 *              Requires json values  `num_frames` (integer)
 *
 * If `rest_mode` is "start" or "step", data generation will not start right away
 * but will wait for a rest trigger providing `num_frames`. In "start" mode this
 * will initiate the stream of data. In "step" mode this will trigger `num_frames`
 * frames to be generated.
 *
 * @author Andre Renard, Kiyoshi Masui
 */


/**
 * @class recvSingleDishVDIF
 * @brief Stage to transmit VDIF data as a UDP stream.
 *
 * This is a producer stage which gathers VDIF-formatted data from a UDP stream and
 * packs it into a target buffer.
 *
 * @par Buffers
 * @buffer out_buf Output kotekan buffer containing VDIF data to be transmitted.
 *     @buffer_format Array of @c uint
 *     @buffer_metadata none
 *
 * @conf   num_freq               Int. Number of time samples to sum.
 * @conf   orig_port              Int. Number of time samples to sum.
 * @conf   orig_server_ip         Int. Number of time samples to sum.
 *
 * @bug    Currently broken, just returns empty frames!
 *
 * @author Andre Renard
 *
 */


/**
 * @class Transpose
 * @brief Generic base class that transposes the data to make time the fastest varying value,
 * compresses it and writes it to a file.
 *
 * All classes which inherit from this should provide the following API:
 *
 *  get_dataset_state(dset_id_t ds_id);
 *  get_frame_data();
 *  create_hdf5_file();
 *  copy_frame_data(uint32_t freq_index, uint32_t time_index);
 *  copy_flags(uint32_t time_index);
 *  write_chunk(size_t t_ind, size_t f_ind);
 *  increment_chunk(size_t &t_ind, size_t &f_ind, bool &t_edge, bool &f_edge);
 *
 * The data is received as one-dimensional arrays
 * that represent flattened-out time-X-frequency matrices. These are transposed
 * and flattened out again to be written to a file. In other words,
 * the transposition makes time the fastest-varying for the data values,
 * where it was frequency before.
 * This stage expects the data to be ordered like RawReader does.
 * Other stages might not guarentee this same order.
 *
 * @warning Don't run this anywhere but on the transpose (gossec) node.
 * The OpenMP calls could cause issues on systems using kotekan pin
 * priority threads (likely the GPU nodes).
 *
 * @par Buffers
 * @buffer in_buf The input stream.
 *         @buffer_format Buffer.
 *         @buffer_metadata Metadata
 *
 * @conf   chunk_size           Array of [int, int, int]. Chunk size of the data
 *                              (freq, prod, time).
 * @conf   outfile              String. Path to the (data-meta-pair of) files to
 *                              write to (e.g. "/path/to/0000_000", without .h5).
 * @conf   comet_timeout        Float, default 60. Timeout for communications with
 *                              dataset broker.
 *
 * @par Metrics
 * @metric kotekan_transpose_data_transposed_bytes
 *         The total amount of data processed in bytes.
 *
 * @author Tristan Pinsonneault-Marotte, Rick Nitsche
 */


/**
 * @class VisWriter
 * @brief Stage to write raw visibility data.
 *
 * This class inherits from the BaseWriter base class and writes raw visibility data
 * @author Richard Shaw and James Willis
 **/


/**
 * @class restInspectFrame
 * @brief Exposes the binary contents of a buffer frame (or subset there of) to
 *        the REST server via a GET request.
 *
 * Returns the latest data copied into the internal frame @c frame_copy.
 * The length of this data is either the frame size, or the @c len config option
 * When returning data to the REST client, this class will not update @c frame_copy
 * So it's possible that if too many requests are made, it will keep returning the
 * same (old) frame contents.  This is a side effect of trying to prevent the system
 * from locking up the buffer itself.  We might want to adjust this someday, but for
 * testing this seems like a reasonable compromise.
 *
 * @par REST Endpoints
 * @endpoint /inspect_frame/\<buffer name\> ``GET`` Returns binary data from the
 *           latest frame in the buffer given in @p in_buf
 *
 * @par Buffers
 * @buffer in_buf Input kotekan buffer
 *     @buffer_format Any
 *     @buffer_metadata Any
 * @conf   len   Int. the amount of bindary data in bytes to return from the
 *               front of the latest frame. Default the frame size of @c in_buf
 *               Note if set to zero, this will be set to frame size of @c in_buf
 *
 * @warning This stage makes a copy of the data in each and every frame ( upto @c len ).
 *          So it should not be used in places where this extra memory copy would be
 *          expensive for the system to deal with, and should only be enabled when it
 *          is needed for trouble shooting.
 *
 * @todo Once the new buffers are implemented this should use the frame "freeze out"
 *       method instead of doing memory copies.
 *
 * @author Andre Renard
 */


/**
 * @class bufferBadInputs
 * @brief Buffers updates to the bad input list.
 *
 * This engine reorders, inverts and generates a mask of bad inputs then stores the mask in a buffer
 *
 * @par Buffers
 * @buffer out_buf Kotekan buffer of bad inputs.
 *     @buffer_format Array of @c uint8_t
 *
 * @conf   updatable_config/bad_inputs  String.  String pointing to the location of the
 *                                      config block containing the following properties:
 *                                      "bad_inputs"  An array of bad inputs in cylinder order.
 *
 * @author James Willis
 *
 */


/**
 * @class BeamExtract
 * @brief Stage for extracting one beam from the beamformer output and putting it into a new frame.
 *
 * Assumes that the number of polarizations is 2
 *
 * @par Buffers
 * @buffer in_buf The GPU beamformer output buffer
 *     @buffer_format Array of 4+4-bit number with the format of [time][beam][pol]
 *     @buffer_metadata chimeMetadata
 *
 * @buffer out_buf The extracted single beam output buffer
 *     @buffer_format Array of 4+4-bit numbers with format [time][pol]
 *     @buffer_metadata BeamMetadata
 *
 * @conf    num_beams             Int.   The total number of beams in the output.
 * @conf    num_pol               Int.   Must be set to 2 to use this stage
 * @conf    extract_beam          Int.   The beam number to extract from the input set of beams;
 *                                       zero based.
 * @conf    samples_per_data_set  Int.   Number of time samples in a data set
 *
 * @author Andre Renard
 */


/**
 * @class removeEv
 * @brief Remove any eigenvalues/vectors from a buffer.
 *
 * @par Buffers
 * @buffer in_buf Input buffer with eigensector.
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 * @buffer out_buf Output buffer without eigensector.
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 *
 * @author Richard Shaw
 */


/**
 * @class hexDump
 * @brief Prints out contents of a buffer in hex in an xxd style format
 *
 * @par Buffers
 * @buffer in_buf The buffer to print the contents of.
 *     @buffer_format any
 *     @buffer_metadata any
 *
 * @conf    len     Default 128.  The number of bytes to print.
 * @conf    offset  Default 0.    The offset into the frame.
 */


/**
 * @class timeDownsample
 * @brief Average a set number of frames on a single-frequency stream to effectively
 *        reduce the cadence of the acquisition.
 *
 * This stage accumulates and averages a specified number of incoming frames on a
 * single-frequency stream to reduce the cadence of the acquisition. Visibilities,
 * eigenvectors, eigenvalues, eigen-rms are averaged. Inverse weights are averaged
 * and divided by number of combined frames to track reduction in variance. Metadata
 * from the first frame is passed on and that of the others discarded.
 * Will throw an exception if more than one frequency is found in the stream.
 *
 * @par Buffers
 * @buffer in_buf The kotekan buffer of the incoming single-frequency stream.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 * @buffer out_buf The kotekan buffer into which low cadence stream is fed.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 *
 * @conf  num_samples  Int. The number of time frames to average.
 * @conf  max_age      Float. How old can a frame be in seconds before it is dropped.
 *                     Default is 120 (i.e. two minutes).
 *
 * @metric  kotekan_timedownsample_skipped_frame_total
 *      The number of frames skipped entirely because they were too old.
 *
 * @author  Tristan Pinsonneault-Marotte
 *
 */


/**
 * @class rfiUpdateMetadata
 * @brief Takes the RFI mask and the lost packet mask buffers and updates the metadata
 *
 * @par Buffers
 * @buffer rfi_mask_buf Mask of RFI flagged samples
 *     @buffer_format Array of @c uint8
 *     @buffer_metadata chimeMetadata
 * @buffer lost_samples_buf Mask of lost samples from packet loss/errors
 *     @buffer_format Array of @c uint8
 *     @buffer_metadata none
 * @buffer gpu_correlation_buf GPU N2 output buffer.  co-producer on this buffer
 *     @buffer_format Array of @c uint
 *     @buffer_metadata chimeMetadata
 *
 * @conf   sk_step               Int   The number of samples zeroed at once as RFI
 * @conf   num_sub_frames        Int   The number sub frames in the GPU stage
 * @conf   samples_per_data_set  Int   Number of samples in the GPU stage input frame
 *
 * @author Andre Renard
 *
 */


/**
 * @class VisSharedMemWriter
 * @brief Export calibration data out to a shared memory region
 *
 * This stage is an improvement upon VisCalWriter. It exports the
 * most recent samples of the calibration data stream to a
 * fixed-length ring-buffer-like shared memory region.
 *
 * The shared memory region is composed of a Structured Data Region
 * (summarises the structure for the data), an Access Record region
 * (records the writer's access to the Data region), and the Data region
 * (ring buffer which contains the data and metadata from the frame).
 *
 * The structure of the shared memory region can be seen in the figure in
 * this issue: https://github.com/kotekan/kotekan/issues/692.
 *
 * The first section of the structured data region contains 0 if if the stage
 * is gracefully shut down. Otherwise, it will contain the number of writes.
 *
 * To ensure the ability to read while the stage is continuously
 * writing to the file, readers can inspect the access record to assess
 * the validity of the data. The access record has a 1:1 correlation to
 * the Data region. Every time the Data is modified, the access record
 * will have the timestamp of modification. While data is written,
 * its access record will be set to -1. Acquiring a semaphore, with the
 * same name as the shared memory region is required before reading
 * from or writing to the access record. The Data region is lock free.
 *
 * This stage writes out the data it receives with minimal processing.
 * Removing certain fields from the output must be done in a prior
 * transformation.
 *
 * A Valve stage should be added before this buffer process
 * to prevent data backing up during semaphore deadlocks.
 *
 * To obtain the metadata about the stream received usage of the datasetManager
 * is required.
 *
 * The dataset ID can be changed in the incoming stream. However, structural
 * parameters that could affect the size of the data cannot be adjusted without
 * triggering a FATAL_ERROR. By default these are `input`, `frequencies`, `products`,
 * `stack`, `eigenvalues`, and `metadata`. This list can be *added* to using the config
 * variable `critical_states`. Any state change not considered critical
 * will continue as normal.
 *
 * @par Buffers
 * @buffer in_buf The buffer streaming data to write
 *          @buffer_format VisBuffer structured
 *          @buffer_metadata VisMetadata
 *
 * @conf    root_path       String. Location in filesystem containing
 *                          shared memory and semaphore.
 * @conf    name           Name of shared memory region and semaphore.
 * @conf    num_samples        Number of time samples stored in ring buffer.
 * @conf    wait_time   Maximum time that the writer will wait to
 *                          unlock a semaphore. If it hits the limit
 *                          Kotekan will shut down the stage.
 * @conf    critical_states List of strings. A list of state types to consider
 *                          critical. That is, if they change in the incoming
 *                          data stream then Kotekan will shut down.
 *
 * @par Metrics
 * @metric dropped_frame_counter
 *          The number of times a frame was dropped because it arrived too late.
 * @metric  access_record_wait_time_seconds
 *          The amount of time the writer spent, in seconds waiting to
 *          access the access record.
 *
 * @author Anja Boskovic
 */


/**
 * @class integratePowerStream
 * @brief A simple stage to take a @c radioPowerStream and sum down.
 *
 * This is a simple signal processing block which takes power data from an input buffer,
 * integrates over time, and stuffs the results into an output buffer.
 * Input and output buffers need not be the same length.
 *
 * @par Buffers
 * @buffer in_buf Input kotekan buffer containing power data to be integrated.
 *     @buffer_format Array of @c uint
 *     @buffer_metadata none
 * @buffer out_buf Output kotekan buffer, where integrated samples will be placed.
 *     @buffer_format Array of @c uint
 *     @buffer_metadata none
 *
 * @conf   integration   Int. Number of time samples to sum.
 *
 * @todo    Update once we have a formal radioPowerStream buffer format.
 * @todo    Add metadata to allow different data types for in/out.
 *
 * @author Keith Vanderlinde
 *
 */


/**
 * @class streamSingleDishVDIF
 * @brief Stage to transmit VDIF data as a UDP stream.
 *
 * This is a consumer stage which takes VDIF-formatted data from a buffer and streams
 * it via UDP to a remote client.
 *
 * @par Buffers
 * @buffer in_buf Input kotekan buffer containing VDIF data to be transmitted.
 *     @buffer_format Array of @c uint
 *     @buffer_metadata none
 *
 * @conf   num_freq               Int. Number of time samples to sum.
 * @conf   dest_port              Int. Number of time samples to sum.
 * @conf   dest_server_ip         Int. Number of time samples to sum.
 *
 * @note    Hasn't been tested lately, should confirm this still works!
 *
 * @author Andre Renard
 *
 */


/*
 * @class rfiBadInputFinder
 * @brief Consumer ``kotekan::Stage`` which consumes a buffer filled with averaged,
 * individual input spectral kurtosis estimates.
 *
 * This stage reads RFI data from a kotekan buffer before packaging it into UDP packets and
 * sending them to a user defined IP address. Each packet is fitted with a header which can be read
 * by the server to ensure that the config parameters of the packet match the server config. This
 * stage simply reads the spectral kurtosis estimates, finds outlier inputs based on the
 * statistics of the estimates, assigns each input a pass or fail for each frame, counts the number
 * of fails for each input, packages the results into a packet (header + data), and sends the
 * packets to a user defined IP address via UDP.
 *
 * UDP packets are packaged in blocks (header + data) where the data is of size
 * num_elements*num_local_freq. The data is the number of statistically un-favourable frames seen
 * from each input in the given time interval. See lib/utils/rfi_functions.h for information
 * regarding packet header.
 *
 * @par Buffers
 * @buffer rfi_in        The kotekan buffer containing spectral kurtosis estimates to be read by the
 * stage.
 * 	@buffer_format   Array of @c floats
 * 	@buffer_metadata chimeMetadata
 *
 * @par REST Endpoints
 * @endpoint    /rfi_broadcast ``POST`` Updates frames per broadcast packet
 *              requires json values      "bi_frames_per_packet"
 *              update config             "bi_frames_per_packet"
 *
 * @conf   num_elements         Int . Number of elements.
 * @conf   num_local_freq       Int . Number of local freq.
 * @conf   num_local_freq       Int (default 1024). Number of total freq.
 * @conf   samples_per_data_set Int . Number of time samples in a data set.
 * @conf   sk_step              Int (default 256). Length of time integration in SK estimate.
 * @conf   frames_per_packet    Int (default 1). The Number of frames to average over before sending
 * a packet
 * @conf   dest_port            Int, The port number for the stream destination (Example: 41214)
 * @conf   dest_server_ip       String, The IP address of the stream destination (Example:
 * 192.168.52.35
 * @conf   dest_protocol        String, Currently only supports 'UDP'
 *
 * @author Jacob Taylor
 */


/**
 * @class HFBTruncate
 * @brief Truncates absorber data and weight values.
 *
 * Absorber values are truncated to a precision based on their
 * weight.
 *
 * @warning Don't run this anywhere but on the transpose (gossec) node.
 * The OpenMP calls could cause issues on systems using kotekan pin
 * priority threads (likely the GPU nodes).
 *
 * @par Buffers
 * @buffer in_buf The input stream.
 *         @buffer_format HFBBuffer.
 *         @buffer_metadata HFBMetadata
 * @buffer out_buf The output stream with truncated values.
 *         @buffer_format HFBBuffer.
 *         @buffer_metadata HFBMetadata
 *
 * @conf   err_sq_lim               Limit for the error of absorber truncation.
 * @conf   weight_fixed_precision   Fixed precision for weight truncation.
 * @conf   data_fixed_precision     Fixed precision for absorber truncation (if
 *                                  weights are zero).
 *
 * @author James Willis
 */


/**
 * @class HFBAccumulate
 * @brief Post-processing engine for output of the CHIME/HFB kernel,
 *        integrates data over 80 frames to create 10s worth of data.
 *        num_beams * num_sub_freq = 1024 * 128
 *
 * This engine sums CHIME/HFB data from 1 GPU stream in each CHIME node,
 * which are stored in the output buffer.
 * Note: _num_frames_to_integrate cannot go below 16 frames as _num_frames_to_integrate cannot be
 * lower than the max_frames_missing
 *
 * This stage will also calculate the within sample variance for weights.
 *
 * The output of this stage is written to a raw file where each chunk is
 * the metadata followed by the frame and indexed by frequency ID.
 * This raw file is then transposed and compressed into a structured
 * HDF5 format by gossec.
 *
 * @par Buffers
 * @buffer hfb_input_buffer Kotekan buffer feeding data from any GPU.
 *     @buffer_format Array of @c floats
 * @buffer cls_buffer Kotekan buffer that contains the compressed lost samples.
 *     @buffer_format Array of @c uint32_t
 * @buffer hfb_out_buf Kotekan buffer that will be populated with integrated data.
 *     @buffer_format Array of @c floats
 *
 * @conf   num_frames_to_integrate  Int. No. of frames to integrate over.
 * @conf   num_frb_total_beams      Int. No. of total FRB beams (should be 1024).
 * @conf   factor_upchan            Int. Upchannelise factor (should be 128).
 * @conf   samples_per_data_set     Int. The number of samples each GPU buffer has
 *                                  been integrated for.
 * @conf   good_samples_threshold   Float. Required fraction of good samples in
 *                                  integration before it is recorded.
 *
 * @author James Willis
 *
 */


/**
 * @class fftwEngine
 * @brief Kotekan Stage to Fourier Transform an input stream.
 *
 * This is a simple signal processing stage which takes (complex) data from an input buffer,
 * Fourier Transforms it with FFTW, and stuffs the results into an output buffer.
 * Both input and output buffers' frame lengths should be integer multiples of the FFT length,
 * though they need not be the same length as each other.
 * Assumes I/Q (complex) data at the input.
 *
 * This producer depends on libfftw3.
 *
 * @par Buffers
 * @buffer in_buf Input kotekan buffer, to be consumed from.
 *     @buffer_format Array of @c shorts
 *     @buffer_metadata none
 * @buffer out_buf Output kotekan buffer, to be produced into.
 *     @buffer_format Array of @c fftwf_complex
 *     @buffer_metadata none
 *
 * @conf   spectrum_length Int. Number of samples in the input spectrum. Defaults to 1024.
 *
 * @todo    Add checking to make sure the input and output buffers' frames are
 *          appropriately sized, i.e. integer multiples of spectrum_length.
 * @todo    Add a flag to allow real inputs.
 * @todo    Add some metadata to allow different data types for in/out.
 *
 * @author Keith Vanderlinde
 *
 */


/**
 * @class HFBTranspose
 * @brief Stage to transpose raw HFB data
 *
 * This class inherits from the Transpose base class and transposes raw HFB data.
 * @author James Willis
 */


/**
 * @class frbPostProcess
 * @brief Post-processing engine for data coming out of the CHIME/FRB kernel stack.
 *
 * This engine gathers CHIME/FRB data from the 4 GPU streams in each CHIME node,
 * packing it into CHIME/FRB L0-L1 packets, which are stored in the output buffer.
 * Prior to packing, the (float) input values are scaled and offset to 8-bit
 * unsigned ints (i.e., 0-255). The scaling is determined on a per-packet basis,
 * using AVX2 instructions to calculate and apply those parameters.
 *
 * This stage can also optionally produce a sum-of-all-beams "incoherent" beam,
 * which will be stored in the 0th beam position in output packets.
 *
 * Time samples with dropped packet are set to zero.
 *
 * This stage depends on ``AVX2`` intrinsics.
 *
 * @par Buffers
 * @buffer in_buf_0 Kotekan buffer feeding data from GPU0.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer in_buf_1 Kotekan buffer feeding data from GPU1.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer in_buf_2 Kotekan buffer feeding data from GPU2.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer in_buf_3 Kotekan buffer feeding data from GPU3.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer out_buf Kotekan buffer that will be populated with packetized data.
 *     @buffer_format Array of @c uchars
 *     @buffer_metadata chimeMetadata
 * @buffer lost_samples_buf Koktekan buffer with drop packet info, where 1=dropped
 *     @buffer_metadata chimeMetadata
 *     @buffer_format Array of @c uint8
 *
 * @conf   num_gpus                   Int. Number of GPUs.
 * @conf   samples_per_data_set       Int. Number of baseband samples corresponding to each buffer.
 * @conf   downsample_time            Int. Number of time samples summed in the FRB kernels.
 * @conf   factor_upchan              Int. Total upchannelization in the FRB kernels.
 * @conf   factor_upchan_out          Int. Upchannelization that exits the FRB kernels,
 *                                        also number of freqs that will go into each output packet
 * @conf   num_beams_per_frb_packet   Int. Number of FRB beam in each output stream. (?) Should be
 * 4?
 * @conf   timesamples_per_frb_packet Int. Number of times that will go into each packet.
 * @conf   incoherent_beam            Bool (default=false). Form the incoherent beam or not.
 *                                        If true, will stuff incoherent beam in position 0.
 * @conf   incoherent_truncate        Float (default=1e10). To deal with inputs / times /freqs with
 *                                        anomalously high values, this limits values used prior to
 * summing into the incoherent beam.
 *
 * @par Metrics
 * @metric kotekan_frb_masked_packets_total
 *         Count of masked packets
 *
 * @author Keith Vanderlinde, Cherry Ng
 *
 */


/**
 * @brief Compress visibility data by stacking together equivalent baselines.
 *
 * This task takes a stream of uncompressed visibility data and performs a
 * (weighted) average over the equivalent baselines.
 *
 * @note This task requires there to be an inputState and prodState registered
 *       for the incoming dataset.
 *
 * @par Buffers
 * @buffer in_bufs The input uncompressed data.
 *         @buffer_format VisBuffer structured.
 *         @buffer_metadata VisMetadata
 * @buffer out_buf The merged and transformed buffer
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 *
 * @conf stack_type             String. Type of stacking to apply to the data.
 *                              Look at documentation of stack_X functions for
 *                              details.
 * @conf exclude_inputs         List of ints. Extra inputs to exclude from
 *                              stack.
 *
 * @par Metrics
 * @metric kotekan_baselinecompression_residuals
 *      The variance of the residuals.
 * @metric kotekan_baselinecompression_time_seconds
 *      The time elapsed to process one frame.
 * @metric kotekan_baselinecompression_frame_total
 *      Number of frames seen by each thread.
 *
 * @author Richard Shaw
 */


/**
 * @brief Copies frames from one buffer into many buffers.
 *
 *
 * @warning The sizes of the frames must be the same in all buffers, and the
 *       metadata types and underlying pools must also be the same.
 *
 *
 * @par Buffer
 * @buffer in_buf Buffer to copy frames from
 *        @buffer_format Matches the output buffers
 *        @buffer_metadata Matches the output buffers
 * @buffer out_bufs Array of output buffers to copy frames into.
 *                 This is a named array of output buffers in the format:
 *                 - buffer_name_0
 *                 - buffer_name_1
 *                 .
 *        @buffer_format any, but all must be the same type.
 *        @buffer_metadata any, but all must be the same type.
 *
 * @author James Willis
 */


/**
 * @class pyPlotN2
 * @brief Consumer ``kotekan::Stage`` to produce PDF plots of correlation matrices.
 *
 * This stage does nothing until it receives a REST request from an outside user.
 * Upon receipt, it spawns a companion python script (``pyPlotN2.py``),
 * and pipes a short configuration header to it, followed by the contents of the next available
 * buffer. The python script generates a pdf plot of the visibilitiy matrix, saving it to a
 * timestamped file.
 *
 * @par REST Endpoints
 * @endpoint /plot_corr_matrix/``gpu_id`` Any contact here triggers a plot dump.
 *
 * @par Buffers
 * @buffer in_buf Buffer containing the data to be plotted. Should be a blocked upper triangle
 * correlation matrix
 *  @buffer_format Array of complex uint32_t values.
 *  @buffer_metadata none
 *
 * @conf gpu_id         Int, used to generate the REST endpoint,
 *                      needed in case of multiple streams in a single kotekan stage.
 *
 * @todo    Make the location of the python plotting script more robust / permanent.
 * @todo    Move config parsing to the constructor.
 * @todo    Spin the triggered dump/executre out into a new thread.
 *
 * @author Keith Vanderlinde
 */


/**
 * @brief Receives frames and metadata from other networked kotekan buffers,
 *        and puts them into a local kotekan buffer.
 *
 * This stage takes frames from one more more sources and places them into the
 * @c buf for use by other local consumer stages. There is no guarantee regarding
 * the order the frames will arrive in.  However all frames will be valid, which is to
 * say they will contain the full set of data sent by the client, or they will not be
 * added to the output buffer.
 *
 * This system works with libevent to do event driven async I/O with worker threads to support
 * higher bandwidth than one thread alone could support.  In libevent terms there is one base
 * thread, and @c num_threads worker threads which handle the libevent callbacks.
 *
 * @par buffers
 * @buffer buf The buffer which accepts new frames (producer)
 *        @buffer_format any
 *        @buffer_metadata any
 *
 * @conf listen_port         Int, default 11024.  The port to listen for new connections
 * @conf num_threads         Int, default 1.  The number of worker threads to use
 * @conf connection_timeout  Int, default 60.  Number of seconds before timeout on transfer
 * @conf drop_frames         Bool, default true.  Whether to drop frames when buffer fills.
 *
 * @par Metrics
 * @metric kotekan_buffer_recv_transfer_time_seconds
 *         The amount of time it took in seconds to transfer the last frame from the
 *         host given by the @c source label
 * @metric kotekan_buffer_recv_dropped_frame_total
 *         The number of times a frame was dropped because the @c buf was full at the time
 *         a block of was aviable to transfer to it.
 *
 * @todo Possibly factor out the threadpool.
 * @todo Allow for a different log level for workers from the main thread.
 *
 * @author Andre Renard
 */


/**
 * @brief Object to store all the details of a connection to a remote
 *        kotekan buffer.
 *
 * Should not be used externally to the @c bufferRecv class.  This is an
 * internal helper class.
 *
 * @author Andre Renard
 */


/**
 * @class BeamInspect
 * @brief Prints out beam information about the frames in the in_buf
 *
 * This class is intended for testing/debugging.
 *
 * @par Buffers
 * @buffer in_buf The buffer to print the contents of.
 *     @buffer_format 4+4-bit complex voltage (beam) data
 *     @buffer_metadata BeamMetadata
 *
 * @author Andre Renard
 */


/**
 * @class DataQuality
 * @brief Compute data quality metric for the visbility data, which is effectively
 *        the total amount of "data weight" per time and frequency."
 *
 * @par Buffers
 * @buffer in_buf Visibility data.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 *
 * @par Metrics
 * @metric kotekan_dataquality_dataquality
 *      The data quality of the visibility data.
 *
 * @author James Willis
 *
 */


/**
 * @brief Drop whole GPU frames based on the high cadence kurtosis statistics.
 *
 * This stage reads the kurtosis data output and uses it to filter the GPU N^2
 * data that feeds through the system. This is complex because the subframe
 * processing of the N^2 data, the kurtosis frames contain the data
 * corresponding to multiple N^2 frames and so we need to be careful to
 * synchronize them.
 *
 * @par Buffers
 * @buffer  in_buf_vis        The kotekan buffer from which frames are read, can be any size.
 *          @buffer_format    GPU packed upper triangle
 *          @buffer_metadata  chimeMetadata
 * @buffer  in_buf_sk         The high cadence Spectral Kurtosis estimates.
 *          @buffer_format    High cadence SK values.
 *          @buffer_metadata  chimeMetadata
 * @buffer  out_buf           The filtered GPU visibility data.
 *          @buffer_format    GPU packed upper triangle
 *          @buffer_metadata  chimeMetadata
 *
 * @conf  num_elements          The number of inputs to the correlator.
 * @conf  num_sub_frames        The number of N^2 frames we will receive per SK frame.
 * @conf  samples_per_data_set  The number of FPGA samples combined into an SK frame.
 * @conf  sk_step               The number of FPGA samples combined into an SK element.
 * @conf  thresholds            List of dicts containing `fraction` and `threshold` keys.
 *                              Each of these is a float giving the *fraction* of SK
 *                              values that must exceed the *threshold* for the frame
 *                              to get dropped.
 *
 * @par Metrics
 * @metric  kotekan_rfiframedrop_failing_frame_total
 *          The number of frames failing each criteria per frequency.
 * @metric  kotekan_rfiframedrop_dropped_frame_total
 *          The number of frames dropped (i.e. failing any single threshold test) for
 *          each frequency.
 * @metric  kotekan_rfiframedrop_frame_total
 *          The total number of frames seen by this stage per frequency.
 *
 * @author  Richard Shaw
 *
 */


/**
 * @class simpleAutocorr
 * @brief Kotekan stage to autocorrelate a single stream of values.
 *
 * This is a simple signal processing stage which takes complex @c float2 data from an input buffer,
 * calculates the modulus squared in each spectral bin, integrates over time,
 * then stuffs the results into an output buffer.
 * Both input and output buffers' frame lengths should be integer multiples of the spectrum length,
 * though they need not be the same length as each other.
 *
 * @par Buffers
 * @buffer in_buf Input kotekan buffer, to be consumed from.
 *     @buffer_format Array of <tt> complex float2 </tt>
 *     @buffer_metadata none
 * @buffer out_buf Output kotekan buffer, to be produced into.
 *     @buffer_format Array of @c uint
 *     @buffer_metadata none
 *
 * @conf   spectrum_length      Int (default 1024). Number of samples in the spectrum.
 * @conf   integration_length   Int (default 1024). Number of time samples to sum.
 *
 * @todo    Convert input buffer to VDIF format?
 * @todo    Add some metadata to allow different data types for in/out.
 *
 * @author Keith Vanderlinde
 *
 */


/*
 * @class rfiVDIF
 * @brief Producer and consumer ``kotekan::Stage`` which consumes input VDIF data and
 * computes spectral kurtosis estimates.
 *
 * This stage is a spectral kurtosis estimator that works on any general kotekan buffer containing
 * VDIF data. This stage move block by block through the VDIF data while computing and integrating
 * power estimates. Once the desired integration length is over, the stage does one of two things
 * (as specified by the user):
 *
 * 1) The stage combines the sums across the element axis and kurtosis values are calculated on
 *the new sum
 *
 * 2) The stage computes kurtosis values for each frequency-element pair
 *
 * There are advantages to both options, however the first is currently heavily favoured by other
 * stages.
 *
 * @par Buffers
 * @buffer vdif_in The kotekan buffer which conatins input VDIF data
 *	@buffer_format	Array of bytes (uint8_t) which conatin VDIF header and data
 *	@buffer_metadata chimeMetadata
 * @buffer rfi_out The kotekan buffer to be filled with kurtosis estimates
 *	@buffer_format Array of floats
 *	@buffer_metadata none
 *
 * @conf rfi_combined   Bool (default true). A flag indicating whether or not to combine data inputs
 * @conf sk_step        Int (default 256). The number of timesteps per kurtosis estimate
 *
 * @author Jacob Taylor
 */


/**
 * @brief Generate a ringmap from a real-time stream of data.
 *
 * Expects frames from the stacked dataset.
 *
 * @par buffers
 * @buffer in_buf The buffer to read from.
 *        @buffer_format VisBuffer
 *        @buffer_metadata VisMetadata
 *
 * @conf feed_sep       Float, default 0.3048. The separation between feeds (in m)
 * @conf apodization    String, default nuttall. The type of window to use for apodization.
 * @conf exclude_autos  Bool, default true. Exclude the autos from the maps.
 *
 *
 * @author Tristan Pinsonneault-Marotte
 */


/**
 * @brief Complete stack over redundant baselines from previously stacked data.
 *
 * This is based on baselineCompression.
 *
 * @todo merge this with baselineCompression
 *
 * @par buffers
 * @buffer in_buf The buffer to read from.
 *        @buffer_format VisBuffer
 *        @buffer_metadata VisMetadata
 * @buffer out_buf The buffer to write to.
 *        @buffer_format VisBuffer
 *        @buffer_metadata VisMetadata
 *
 *
 * @author Tristan Pinsonneault-Marotte
 */


/**
 * @class InputSubset
 * @brief ``kotekan::Stage`` that extracts a subset of the inputs.
 *
 * The difference between this stage and the only_inputs option of prodSubset
 * is that this will also subset the `num_elements` sized parts of the frames,
 * such as gains and eigenvectors.
 *
 * @par Buffers
 * @buffer in_buf The kotekan buffer from which the visibilities are read, can be any size.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 * @buffer out_buf The kotekan buffer which will be fed the subset of visibilities.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 *
 * @conf  inputs  List of ints. The channel IDs for the inputs to extract.
 *
 * @author Richard Shaw
 */


/**
 * @class VisTranspose
 * @brief Stage to transpose raw visibility data
 *
 * This class inherits from the Transpose base class and transposes raw visibility data.
 * @author Tristan Pinsonneault-Marotte, Rick Nitsche
 */


/**
 * @class ReadGain
 * @brief read in new gain files for FRB/Tracking beamformer when available
 *
 * @par Buffers
 * @buffer gain_frb_buf Array of gains size 2048*2
 *     @buffer_format Array of @c floats
 *     @buffer_metadata none
 * @buffer gain_tracking_buf Array of gains size 2048*2*nbeam
 *     @buffer_format Array of @c floats
 *     @buffer_metadata none
 *
 * @conf   num_elements         Int (default 2048). Number of elements
 * @conf   num_pulsar           Int (default 10). Number of pulsar beams to be formed
 * @conf   scaling              Float (default 1.0). Scaling factor on gains
 * @conf   default_gains        Float array (default 1+1j). Default gain value if gain file is
 * missing
 *
 * @par Metrics
 * @metric kotekan_gains_last_update_success
 *     Flag with value 1 if the last gains update succeeded, 0 if failed.
 *     Labels: `type` (values: "frb", "pulsar")
 * @metric kotekan_gains_last_update_timestamp
 *     Timestamp of last gains update.
 *     Labels: `type` (values: "frb", "pulsar")
 *
 * The gain path is registered as a subscriber to an updatable config block.
 * For the FRB, it is one directory path: '{"frb_gain_dir":"the_new_path"}'
 * For the tracking beamformer, it is an array of @c num_beams paths for each of the @c num_beams
 * beams:
 * '{"pulsar_gain_dir":["path0","path1","path2","path3","path4","path5","path6","path7","path8","path9"]}'
 *
 * @author Cherry Ng
 *
 */


/**
 * @class ExampleDotProduct
 * @brief A stage to compute the dot product between two vectors: A and B, which are represented by
 * two buffers. The result is written to an output buffer.
 *
 * @par Buffers
 * @buffer in_a_buf The input buffer representing vector A.
 *      @buffer_format Array of floats
 *      @buffer_metadata none
 * @buffer in_b_buf The input buffer representing vector B.
 *      @buffer_format Array of floats
 *      @buffer_metadata any
 * @buffer out_buf The output buffer to hold the result.
 *      @buffer_format Array of floats
 *      @buffer_metadata any
 */


/**
 * @class ExampleProducer
 * @brief An example producer stage that sets each element of a buffer to a constant value.
 *
 * @par Buffers
 * @buffer out_buf The buffer to process the contents of.
 *      @buffer_format any
 *      @buffer_metadata any
 *
 * @conf    init_value  Default 0.  The value to set each element to.
 */


/**
 * @class ExampleConsumer
 * @brief An example consumer stage to print the contents of a buffer.
 *
 * @par Buffers
 * @buffer in_buf The buffer to process the contents of.
 *      @buffer_format any
 *      @buffer_metadata any
 *
 */


/**
 * @class rawFileRead
 * @brief Read and stream a dumped buffer
 *
 * @par Buffers
 * @buffer buf The data read from the raw file.
 *         @buffer_format   Any
 *         @buffer_metadata Any
 *
 * @conf   base_dir         String. Directory to read from.
 * @conf   file_name        String. Base filename to read.
 * @conf   file_ext         String. File extension.
 * @conf   end_interrupt    Bool. Interrupt Kotekan if run out of files to read
 */


/*
 * @class rfiAVXVDIF
 * @brief Consumer ``kotekan::Stage`` which consumer a buffer filled with VDIF data and
 * produces a buffer filled with spectral kurtosis estimates.
 *
 * This stage read input VDIF data and computes spectral kurtosis estimates at a variable time
 * cadence. The estimates are computed through AVX2 intrinsics. The data is read in sections of 64
 * 4-bit numbers before the power and square power are computed and stored. After a certain amount
 * of time, the integrated values are turned into a single spectral kurtosis estimate and added to
 * the output array.
 *
 * @par Buffers
 * @buffer vdif_in      The kotekan buffer containing VDIF input data.
 *      @buffer_format  Array of @c uint8_t
 *      @buffer_metadata chimeMetadata
 * @buffer rfi_out      The kotekan buffer to be filled with spectral kurtosis estimates.
 *      @buffer_format  Array of @c float
 *      @buffer_metadata chimeMetadata
 *
 * @conf   num_elements         Int (default 2048). Number of elements.
 * @conf   num_local_freq       Int (default 1). Number of local freq.
 * @conf   samples_per_data_set Int (default 32768). Number of time samples in a data set.
 * @conf   sk_step              Int Length of time integration in SK estimate.
 * @conf   frames_per_packet    Int The Number of frames to average over before sending each UDP
 * packet.
 * @conf   rfi_combined         Bool Whether or not the kurtosis measurements include an input sum.
 *
 * @author Jacob Taylor
 */


/**
 * @class FEngine
 * @brief A stage that simulates the CHORD F-engine hardware and a simple scene (eg, a
 * single radio source), producing data useful for end-to-end testing of our kernels.
 * (The actual computational core is implemented in Julia, in FEngine.jl)
 *
 * @conf num_components Int (=2) complex components
 * @conf num_polations  Int (=2) polarizations
 * @conf source_amplitude  Float  simulated radio source amplitude
 * @conf source_frequency  Float  simulated radio source frequency (in Hz)
 * @conf source_position_x  Float  simulated radio source position (east-west angle in radians)
 * @conf source_position_y  Float  simulated radio source position (north-south angle in radians)
 * @conf num_dish_locations_M Int  east-west dish grid size
 * @conf num_dish_locations_N Int  north-south dish grid size
 * @conf dish_separation_x  Float east-west dish separation (in meters)
 * @conf dish_separation_y  Float north-south dish separation (in meters)
 * @conf num_dishes         Int   number of dishes
 * @conf dish_locations     vector of Int  A pair of integers for each dish, giving its position in
 * the M,N grid.
 * @conf adc_frequency      Float  how many samples per second does the ADC perform?
 * @conf num_taps           Int    how many taps in the F-engine's polyphase filter bank?
 * @conf num_frequencies    Int    how many frequencies are produced by the F-engine?
 * @conf num_times          Int    how many time samples per chunk?
 * @conf bb_num_dishes_M    Int    Baseband beamformer: input dish grid size
 * @conf bb_num_dishes_N    Int    Baseband beamformer: input dish grid size
 * @conf bb_num_beams_P    Int    Baseband beamformer: output beam grid size
 * @conf bb_num_beams_Q    Int    Baseband beamformer: output beam grid size
 * @conf bb_beam_separation_x Float Baseband beamformer: output beam spacing east-west (radians?)
 * @conf bb_beam_separation_y Float Baseband beamformer: output beam spacing north-south (radians?)
 * @conf upchannelization_factor Int Upchan
 * @conf num_frames              Int how many frames of data to produce
 */


/**
 * @class freqSplit
 * @brief Separate a VisBuffer stream into two by selecting frequencies in the upper and lower
 * half of the band.
 *
 * This task takes data coming out of a VisBuffer stream and separates it into
 * two streams. It selects which frames to copy to which buffer by assigning
 * frequencies in the upper and lower half of the CHIME band to different buffer
 * streams.
 *
 * @par Buffers
 * @buffer in_buf The buffer to be split
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 * @buffer out_bufs The two buffers containing the respective upper or lower band frequencies
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 *
 * @conf split_freq             Double. Frequency to split the incoming buffer
 *                              at. Lower frequencies got to the first output
 *                              buffer, equal and higher frequencies go to the
 *                              second. Default 512.
 *
 * @todo Generalise to arbitary frequency splits.
 * @author Mateus Fandino
 */


/**
 * @class eigenVis
 * @brief Perform eigen factorization of the visibilities
 *
 * This task performs the factorization of the visibility matrix into
 * ``num_ev`` eigenvectors and eigenvalues and stores them in reserve space
 * in the ``VisBuffer``. They are stored in descending order of the eigenvalue.
 *
 * @par Buffers
 * @buffer in_buf The set of buffers coming out the GPU buffers
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 * @buffer out_buf The merged and transformed buffer
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 *
 * @conf  num_elements          Int. The number of elements (i.e. inputs) in the
 *                              correlator data.
 * @conf  block_size            Int. The block size of the packed data.
 * @conf  num_ev                UInt. The number of eigenvectors to be calculated as
 *                              an approximation to the visibilities.
 * @conf  num_diagonals_filled  Int, default 0. Number of diagonals to fill with
 *                              the previous time step's solution prior to
 *                              factorization. For example, setting to 1 will replace
 *                              the main diagonal only. Filled with zero on the first
 *                              time step.
 * @conf  exclude_inputs        List of UInts, optional. Inputs to exclude (rows and
 *                              columns to set to zero) in visibilities prior to
 *                              factorization.
 *
 * @par Metrics
 * @metric kotekan_eigenvis_comp_time_seconds
 *         Time required to find eigenvectors. An exponential moving average over
 *         ~10 samples.
 * @metric kotekan_eigenvis_eigenvalue
 *         The value of each eigenvalue calculated, or the RMS.
 * @metric kotekan_eigenvis_lapack_failure_total
 *         The number of frames skipped due to LAPACK failing (because of bad input data
 *         or other reasons).
 *
 * @author Kiyoshi Masui
 */


/**
 * @class VisRawReader
 * @brief Stage to read raw visibility data.
 *
 * This class inherits from the RawReader base class and reads raw visibility data
 * @author Richard Shaw, Tristan Pinsonneault-Marotte, Rick Nitsche
 */


/**
 * @class BadInputFlag
 * @brief searches for bad inputs that aren't marked in the `flags` field.
 *
 * @par Buffers
 * @buffer in_buf The buffer from which the visibilities are checked, must be a full triangle.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 * @buffer out_buf The kotekan buffer which will be fed the output.
 *     @buffer_format VisBuffer structured
 *     @buffer_metadata VisMetadata
 *
 * @par Metrics
 * @metric kotekan_badinputflag_frames_total
 *      The number of frames found with unmarked bad inputs. The metrics are labelled with
 *      which `input` is bad, and why (Infinite weight, NaN weight).
 *
 * @warning  This will only work correctly if the full correlation triangle is
 *           passed in as input.
 *
 * @author  Richard Shaw
 *
 */


/*
 * @class rfiBroadcast
 * @brief Consumer ``kotekan::Stage`` which consumes a buffer filled with spectral kurtosis
 * estimates.
 *
 * This stage reads RFI data from a kotekan buffer before packaging it into UDP packets and
 * sending them to a user defined IP address. Each packet is fitted with a header which can be read
 * by the server to ensure that the config parameters of the packet match the server config. This
 * stage simply reads the spectral kurtosis estimates, averages them for a single frame, averages
 * frames_per_packet frames together, packages the results into a packet (header + data), and sends
 * the packets to a user defined IP address via UDP.
 *
 * @par Buffers
 * @buffer rfi_in	The kotekan buffer containing spectral kurtosis estimates to be read by the
 * stage.
 * 	@buffer_format	Array of @c floats
 * 	@buffer_metadata chimeMetadata
 *
 * @par REST Endpoints
 * @endpoint    /rfi_broadcast ``POST`` Updates frames per broadcast packet
 *              requires json values      "frames_per_packet"
 *              update config             "frames_per_packet"
 *
 * @conf   num_elements         Int . Number of elements.
 * @conf   num_local_freq       Int . Number of local freq.
 * @conf   num_total_freq       Int (default 1024). Number of total freq.
 * @conf   samples_per_data_set Int . Number of time samples in a data set.
 * @conf   sk_step              Int (default 256). Length of time integration in SK estimate.
 * @conf   frames_per_packet    Int (default 1). The Number of frames to average over before sending
 * each UDP pack$
 * @conf   rfi_combined         Bool (default true). Whether or not the kurtosis measurements
 * include an input sum.
 * @conf   total_links          Int (default 1). Number of FPGA links per buffer
 * @conf   dest_port            Int, The port number for the stream destination (Example: 41214)
 * @conf   dest_server_ip       String, The IP address of the stream destination (Example:
 * 192.168.52.174)
 * @conf   dest_protocol        String, Currently only supports 'UDP'
 *
 * @author Jacob Taylor
 */


/**
 * @class BaseWriter
 * @brief Generic class to write a buffer as raw data.
 *
 * All classes which inherit from this should provide the following API:
 *
 * make_metadata(dset_id_t ds_id);
 * get_dataset_state(dset_id_t ds_id);
 * write_data(const FrameView& frame);
 *
 * This stage writes out the data it receives with minimal processing.
 * Removing certain fields from the output must be done in a prior
 * transformation. See `removeEv` for instance.
 *
 * To obtain the metadata about the stream received usage of the datasetManager
 * is required.
 *
 * This stage will check that git hash of the data source (obtained from the
 * metadataState) matches the current version. Depending on the value of
 * `ignore_version` a mismatch will either generate a warning, or cause the
 * mismatched data to be dropped.
 *
 * The output files will write out the dataset ID of every piece of data
 * written. This allows the dataset ID to change on the incoming stream without
 * requiring a new file or acquisition to be started. However, some types of
 * state change will always cause a new acquisition to be started. By default
 * these are the structural parameters `input`, `frequencies`, `products`, and
 * `stack` along with `gating` and `metadata`. This list can be *added* to
 * using the config variable `critical_states`. Any state change not considered
 * critical will cause an updated ID to be written into the file, but the
 * acquisition will continue as normal.
 *
 * The output is written a specified format. Either the CHIME N^2 HDF5 format
 * version 3.1.0, or the raw format which can be processed into that format by
 * gossec.
 *
 * @par Buffers
 * @buffer in_buf The buffer streaming data to write
 *         @buffer_format VisBuffer structured
 *         @buffer_metadata VisMetadata
 *
 * @conf   file_type        String. Type of file to write. One of 'hdf5',
 *                          'hdf5fast' or 'raw'.
 * @conf   root_path        String. Location in filesystem to write to.
 * @conf   instrument_name  String (default: chime). Name of the instrument
 *                          acquiring data (if ``node_mode`` the hostname is
 *                          used instead)
 * @conf   file_length      Int (default 1024). Maximum number of samples to
 *                          write into a file.
 * @conf   window           Int (default 20). Number of samples to keep active
 *                          for writing at any time.
 * @conf   acq_timeout      Double (default 300). Close acquisitions when they
 *                          have been inactive this long (in seconds).
 * @conf   ignore_version   Bool (default False). If true, a git version
 *                          mistmatch will generate a warning, if false, it
 *                          will cause data to be dropped. This should only be
 *                          set to true when testing.
 * @conf   critical_states  List of strings. A list of state types to consider
 *                          critical. That is, if they change in the incoming
 *                          data stream then a new acquisition will be started.
 *
 * @par Metrics
 * @metric kotekan_writer_write_time_seconds
 *         The write time of the raw writer. An exponential moving average over ~10
 *         samples.
 * @metric kotekan_writer_late_frame_total
 *         The number of frames dropped while attempting to write as they are too late.
 * @metric kotekan_writer_bad_dataset_frame_total
 *         The number of frames dropped as they belong to a bad dataset.
 *
 * @author Richard Shaw and James Willis
 **/


/**
 * @brief Selects buffers based on the values in an updatable config endpoint.
 *
 * An example config:
 *
 * buffer_switch:
 * kotekan_stage: bufferSwitch
 * in_bufs:
 *   - network_data_0: gpu_data_buffer_0
 *   - network_data_1: gpu_data_buffer_1
 * out_buf: network_buffer
 * updatable_config: "/buffer_switch/switch_status"
 * switch_status:
 *   kotekan_update_endpoint: "json"
 *   network_data_0: false # Don't merge frames from gpu_data_buffer_0
 *   network_data_1: true  # Merge frames from gpu_data_buffer_1
 *
 * See bufferMerge for more docs.  Requires internal names.
 *
 * @conf updatable_config  String.  JSON pointer to the updatable config block.
 *                                  An example block would be:
 *                                  switch_config:
 *                                      updatable_config: "json"
 *                                      internal_buffer_name_0: true
 *                                      internal_buffer_name_1: false
 *
 * @author Andre Renard
 */


/**
 * @class DPDKShuffleSimulate
 * @brief Simulates the 4-way DPDK shuffle for ICEBoard data
 *
 * Note this currently only simulates the data rate, not the actual data the ICEBoards generate
 *
 * @par Buffers
 * @buffer voltage_data_buf_N The buffers to store the voltage data from the FPGAs
 *     @buffer_format 4+4-bit complex number
 *     @buffer_metadata chimeMetadata
 *
 * @buffer lost_samples_buf Buffer of flags set to 1 if the corresponding sample in the voltage
 *                          buffer was lost.
 *     @buffer_format uint8_t flags
 *     @buffer_metadata chimeMetadata
 *
 * @conf    samples_per_data_set  The number of samples in each frame.
 *
 * @todo Add an option to generate actual simulated voltage data
 *
 * @author Andre Renard
 */


/**
 * @brief Delays release of frames until the buffer is sufficiently full.
 *
 *
 * @par Buffer
 * @buffer in_buf Buffer to delay frames from
 *        @buffer_format Matches the output buffer
 *        @buffer_metadata Matches the output buffer
 * @buffer out_buf Buffer to release frames to
 *        @buffer_format Matches the input buffer
 *        @buffer_metadata Matches the input buffer
 *
 * @conf   copy_frame Bool. Flag to copy or swap frames
 * @conf   num_frames_to_hold Int. No. of frames to hold in buffer before releasing one
 *
 * @author James Willis
 */


/**
 * @class BasebandWriter
 * @brief
 *
 * @par Buffers
 * @buffer in_buf The buffer streaming data to write
 *         @buffer_format BasebandBuffer structured
 *         @buffer_metadata BasebandMetadata
 *
 * @conf   root_path        String. Location in filesystem to write to.
 *
 * @conf   dump_timeout     Double (default 60). Close dump files when they
 *                          have been inactive this long (in seconds).
 *
 * @conf   max_frames_per_second Double (default 0) Maximum throughput in
 *                          frames/s at which data is taken out of the input
 *                          buffer. Value of 0 or less disabled the throttling.
 *
 * @par Metrics
 * @metric kotekan_baseband_writeout_in_progress
 *         Set to 1 when a frequency is being written to, 0 otherwise.
 *
 * @metric kotekan_baseband_writeout_active_events
 *         The number of events with any raw files still open
 *
 * @metric kotekan_writer_write_time_seconds
 *         The write time of the raw writer. An exponential moving average over ~10
 *         samples.
 *
 * @metric kotekan_writer_bytes_total
 *         Number of bytes written to files since the start of this stage
 *
 */


/**
 * @class visAccumulate
 * @brief Accumulate the high rate GPU output into integrated VisBuffers.
 *
 * This stage will accumulate the GPU output and calculate the within sample
 * variance for weights.
 *
 * It tags the stream with a properly allocated dataset_id and adds associated
 * datasetStates to the datasetManager.
 *
 * @par Buffers
 * @buffer in_buf
 *         @buffer_format GPU packed upper triangle
 *         @buffer_metadata chimeMetadata
 * @buffer out_buf The accumulated and tagged data.
 *         @buffer_format VisBuffer structured.
 *         @buffer_metadata VisMetadata
 *
 * @conf  samples_per_data_set  Int. The number of samples each GPU buffer has
 *                              been integrated for.
 * @conf  num_gpu_frames        Int. The number of GPU frames to accumulate over.
 * @conf  integration_time      Float. Requested integration time in seconds.
 *                              This can be used as an alterative to
 *                              `num_gpu_frames` (which it overrides).
 *                              Internally it picks the nearest acceptable value
 *                              of `num_gpu_frames`.
 * @conf  num_elements          Int. The number of elements (i.e. inputs) in the
 *                              correlator data.
 * @conf  num_freq_in_frame     Int. Number of frequencies in each GPU frame.
 * @conf  block_size            Int. The block size of the packed data.
 * @conf  input_reorder         Array of [int, int, string]. The reordering mapping.
 *                              Only the first element of each sub-array is used and
 *                              it is the the index of the input to move into this
 *                              new location. The remaining elements of the subarray
 *                              are for correctly labelling the input in
 *                              ``VisWriter``.
 * @conf  low_sample_fraction   If a frames has less than this fraction of the
 *                              data expected, skip it. This is set to 1% by default.
 * @conf  instrument_name       String. Name of the instrument. Default "chime".
 * @conf  freq_ids              Vector of UInt32. Frequency IDs on the stream.
 *                              Default 0..1023.
 * @conf  max_age               Float. Drop frames later than this number of seconds.
 *                              Default is 60.0
 *
 * @par Metrics
 * @metric  kotekan_visaccumulate_skipped_frame_total
 *      The number of frames skipped entirely because they were under the
 *      low_sample_fraction, or too old.
 *
 * @author Richard Shaw, Tristan Pinsonneault-Marotte
 */


/**
 * @class HFBRawReader
 * @brief Read and stream a raw 21cm absorber file.
 *
 * This class inherits from the RawReader base class and reads raw 21cm absorber data
 * @author James Willis
 */


/**
 * @class pulsarPostProcess
 * @brief Post-processing engine for output of the CHIME/Pulsar kernel,
 *        makes packets with either 625 or 3125 time samples.
 *
 * This engine gathers CHIME/Pulsar data from the 4 GPU streams in each CHIME node,
 * which are stored in the output buffer.
 * There are two accepted configurations:
 * -- packet with 625 time samples: each packet consists of 4 freq, 2 pol,
 *    and one beam (4f-625t-2p-1b), where freq is the fastest varying index.
 *    All 4 freq are assumed to be observing the same 10 pulsars.
 * -- packet with 3125 time samples: each packet consists of 1 freq, 2 pol,
 *    and one beam (1f-3125t-2p-1b), where freq is the fastest varying index.
 *    A padding of 6-B is required, making the 3125-format not strictly VDIF, hence undesirable.
 * In both cases, the header contains 32-B.
 * Prior to packing, the real and imag part of the (float) input values are scaled and
 * offset to 4-bit unsigned ints (i.e., 0-15) independently.
 * The scaling factor is an static input value provided by the scheduler,
 * and can be different on a per beam basis.
 *
 * @par Buffers
 * @buffer network_input_buffer_0 Kotekan buffer feeding data from GPU0.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer network_input_buffer_1 Kotekan buffer feeding data from GPU1.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer network_input_buffer_2 Kotekan buffer feeding data from GPU2.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer network_input_buffer_3 Kotekan buffer feeding data from GPU3.
 *     @buffer_format Array of @c floats
 *     @buffer_metadata chimeMetadata
 * @buffer pulsar_out_buf Kotekan buffer that will be populated with packetized data.
 *     @buffer_format Array of @c uint
 *     @buffer_metadata chimeMetadata
 *
 * @conf   num_gpus             Int. No. of GPUs.
 * @conf   samples_per_data_set Int. No. of baseband samples corresponding to each buffer.
 * @conf   num_pulsar           Int. No. of total pulsar beams (should be 10).
 * @conf   num_pol              Int. No. of polarization (should be 2).
 * @conf   timesamples_per_pulsar_packet    Int. Number of times that will go into each packet.
 * (should be 3125 or 625)
 *
 * @author Cherry Ng
 *
 */