// Copyright (c) 2020 Cisco Systems and/or its affiliates.
// Licensed under the Apache License, Version 2.0 (the "License");
// that can be found in the LICENSE file in the root of the source
// tree.

package field_engine

import (
	"encoding/binary"
	"errors"
	"fmt"
	"math"
	"math/rand"
	"net/url"
	"strings"

	"github.com/intel-go/fastjson"
)

/* Field Engine offers the possibility to manipulate packets and parts of packets
by generating values in different forms and operations.
An engine has a small context and will generate values from that context */

// FieldEngineIF is a interface that every type of engine/generator should implement
// in order to provive common functionality to the caller. A caller doesn't care which
// type of field he is updating, but all the types must offer common functionality.
type FieldEngineIF interface {
	// Update updates the byte slice it receives with the new generated value.
	// It writes from the beginning of the slice.
	// If the length of the slice is shorter that the length of the variable we are
	// trying to write, it will return an error.
	// It is the responsibility of the caller to provide Update with a long enough
	// slice. It returns the number of bytes it wrote as the first integer parameter.
	Update(b []byte) (int, error)
	// GetOffset returns the offset of the packet as the interface was provided with.
	// The caller should use GetOffset to provide the interface with the correct
	// byte slice.
	GetOffset() uint16
	// GetSize() returns the size of the variable that the engine will write in
	// the slice byte the next time it will be called. In order to provide a slice
	// long enough, the caller should use GetSize.
	GetSize() uint16
}

// HistogramEntry is an interface for generic types of Histogram Engines.
// HistogramEngine is a fast non uniform pseudo random generator that can generate
// different types of elements. Each entry in this histogram must provide a probability
// for that entry to be chosen and a value for the histogram engine to output.
type HistogramEntry interface {
	// GetProb returns the probability of this entry to be chosen.
	GetProb() uint32
	// GetValue creates a byte buffer with the value that this entry outputs
	// in case it is picked. This value will be generated by the engine.
	// In case the input is incorrect and a value can't be generated, it will
	// return an error.
	GetValue(size uint16) ([]byte, error)
}

// maxUInt64 calculates the max between 2 uint64.
func maxUInt64(a, b uint64) (max uint64) {
	if a >= b {
		max = a
	} else {
		max = b
	}
	return max
}

// maxInt64 calculates the max between 2 uint64.
func maxInt64(a, b int64) (max int64) {
	if a >= b {
		max = a
	} else {
		max = b
	}
	return max
}

// calcDomainLenInt calculates the domain length between two integers min and max,
// meaning the number of integers between min and max including min and max.
// Assumes min < max.
func calcDomainLenInt(min int64, max int64) uint64 {
	if min == math.MinInt64 && max == math.MaxInt64 {
		// Domain Length is MaxUint64 + 1, can't be represented with uint64, but MaxUint64 is close enough.
		return math.MaxUint64
	}
	var domainLen uint64
	if min > 0 {
		// can't overflow int64 since min > 0
		// 0 < min < max
		domainLen = uint64(max - min + 1)
	} else {
		if max > 0 {
			// min < 0 < max
			domainLen = uint64(max) + uint64(min*-1) + 1
		} else {
			// min < max < 0
			domainLen = uint64(min*-1) - uint64(max*-1) + 1
		}
	}
	return domainLen
}

// findValue returns the index of the val item in array arr if it is found and true,
// else -1 and false.
func findValue(arr []uint16, val uint16) (int, bool) {
	for i, item := range arr {
		if item == val {
			return i, true
		}
	}
	return -1, false
}

// PutValue puts value in buffer depending on the size.
func PutValue(size uint16, value uint64, b []byte, mgr *FieldEngineManager) error {
	if len(b) < int(size) {
		mgr.counters.bufferTooShort++
		return fmt.Errorf("Provided slice is shorter that the size of the variable to write, want at least %v, have %v.", size, len(b))
	}
	switch size {
	case 1:
		b[0] = uint8(value)
	case 2:
		binary.BigEndian.PutUint16(b, uint16(value))
	case 4:
		binary.BigEndian.PutUint32(b, uint32(value))
	case 8:
		binary.BigEndian.PutUint64(b, value)
	default:
		mgr.counters.invalidSize++
		return errors.New("Size should be 1, 2, 4 or 8.")
	}
	return nil
}

/* ------------------------------------------------------------------------------
								BaseNumericEngineParams
--------------------------------------------------------------------------------*/
// BaseNumericEngineParams is a struct that gathers base params for uint/int engines.
type BaseNumericEngineParams struct {
	Size   uint16 `json:"size"`   // Size of the uint variable in bytes
	Offset uint16 `json:"offset"` // Offset in which to write in the packet
	Op     string `json:"op"`     // Operation which provides the generation, can be {inc, dec, rand}
	Repeat uint16 `json:"repeat"` // Number of times to repeat current value. Default = 1.
	Step   uint64 `json:"step"`   // Step to decrement or increment, rand will be ignored. Default=1.
}

// GetOffset implements the GetOffset function of FieldEngineIF.
func (o *BaseNumericEngineParams) GetOffset() uint16 {
	return o.Offset
}

// GetSize implements the GetSize function of FieldEngineIF.
func (o *BaseNumericEngineParams) GetSize() uint16 {
	return o.Size
}

/* ------------------------------------------------------------------------------
								UInt Engine
--------------------------------------------------------------------------------*/
// UIntEngineParams is a struct of parameters for the UIntEngine.
type UIntEngineParams struct {
	BaseNumericEngineParams
	MinValue  uint64 `json:"min"`  // Minimal value of the domain
	MaxValue  uint64 `json:"max"`  // Maximal value of the domain
	InitValue uint64 `json:"init"` // Initial value in the generator, default = min value.
}

// UIntEngine is a field engine which is responsible to generate variables of uint types.
// These types can be of lengths 1, 2, 4, 8 bytes (uint8, uint16, uint32, uint64).
// The next variable can be generated through different operations, an increment of the current value,
// a decrement of the current value, or some random generation.
type UIntEngine struct {
	*UIntEngineParams                     // Pointer to params as provided by the caller
	currValue         uint64              // Current value in the generator
	domainLen         uint64              // Domain length
	repeatCount       uint16              // Number of times current value was repeated
	mgr               *FieldEngineManager // Field engine manager
}

// CreateUIntEngine creates a new FieldEngineIF interface of type UIntEngine.
func CreateUIntEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// Parse the params.
	p := UIntEngineParams{}
	p.Step = 1
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create and return new engine
	return NewUIntEngine(&p, mgr)
}

// NewUintEngine creates a new UIntEngine.
func NewUIntEngine(params *UIntEngineParams, mgr *FieldEngineManager) (*UIntEngine, error) {
	o := new(UIntEngine)
	o.mgr = mgr
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.UIntEngineParams = params
	o.domainLen = (o.MaxValue - o.MinValue + 1)
	if o.domainLen == 0 {
		// when min = 0 and max = MaxUint64 domainLen can't be represented on a uint64.
		// So we try to get as close as we can.
		o.domainLen = math.MaxUint64
	}
	if o.Step > o.domainLen {
		o.Step = o.Step % o.domainLen
	}
	o.currValue = maxUInt64(o.MinValue, o.InitValue)
	return o, nil
}

// ValidateParams validates the parameters of the UIntEngine.
func (o *UIntEngine) validateParams(params *UIntEngineParams) (err error) {
	err = nil
	if params.MinValue > params.MaxValue {
		err = fmt.Errorf("Min value %v is bigger than max value %v.", params.MinValue, params.MaxValue)
		o.mgr.counters.maxSmallerThanMin++
	}
	if params.InitValue != 0 && (params.InitValue < params.MinValue || params.InitValue > params.MaxValue) {
		err = fmt.Errorf("Init value %v must be between [%v - %v].", params.InitValue, params.MinValue, params.MaxValue)
		o.mgr.counters.badInitValue++
	}
	sizes := []uint16{1, 2, 4, 8}
	maxPossible := []uint64{math.MaxUint8, math.MaxUint16, math.MaxUint32, math.MaxUint64}
	i, ok := findValue(sizes, params.Size)
	if !ok {
		err = fmt.Errorf("Invalid size %v. Size should be {1, 2, 4, 8}.", params.Size)
		o.mgr.counters.invalidSize++
	} else {
		if params.MaxValue > maxPossible[i] {
			err = fmt.Errorf("Max value %v cannot be represented with size %v.", params.MaxValue, params.Size)
			o.mgr.counters.sizeTooSmall++
		}
	}
	if params.Op != "inc" && params.Op != "dec" && params.Op != "rand" {
		err = fmt.Errorf("Unsupported operation %v.", params.Op)
		o.mgr.counters.badOperation++
	}
	return err
}

// IncValue increments the value according to step
func (o *UIntEngine) IncValue() {
	// Need to be very careful here with overflows.
	left := o.MaxValue - o.currValue // this will never overflow as currValue < maxValue
	if o.Step <= left {
		// simple increment by step, not overflow of domain
		// step is fixed module size of domain
		o.currValue += o.Step
	} else {
		// overflow of domain
		// if here then (step > left) therefore step - left - 1 will not overflow
		o.currValue = o.MinValue + (o.Step - left - 1) // restart also consumes 1
	}
}

// DecValue decrements the value according to step
func (o *UIntEngine) DecValue() {
	left := o.currValue - o.MinValue // this will never overflow as currValue > minValue
	if o.Step <= left {
		// no overflow of domain
		// step is fixed module size of domain
		o.currValue -= o.Step
	} else {
		// overflow of domain
		// if here then (step > left) therefore step - left - 1 will not overflow
		o.currValue = o.MaxValue - (o.Step - left - 1) // restart also consumes 1
	}
}

// RandValue generates a random value in the domain [min, max]
func (o *UIntEngine) RandValue() {
	// Generates a uint64 with uniform distribution.
	// Converts the generated value to a value in the domain by adding the modulus of domainLength
	// to the minimal value.
	genValue := rand.Uint64()
	o.currValue = o.MinValue + (genValue % o.domainLen)
}

// PerformOp performs the operation, either it is rand, inc or dec.
func (o *UIntEngine) PerformOp() (err error) {
	err = nil
	switch o.Op {
	case "inc":
		o.IncValue()
	case "dec":
		o.DecValue()
	case "rand":
		o.RandValue()
	default:
		o.mgr.counters.badOperation++
		err = errors.New("Unrecognized operation")
	}
	return err
}

// Update implements the Update function of FieldEngineIF.
func (o *UIntEngine) Update(b []byte) (int, error) {

	err := PutValue(o.Size, o.currValue, b, o.mgr)
	if err != nil {
		// errors already set
		return 0, err
	}

	o.repeatCount += 1
	if o.repeatCount >= o.Repeat {
		o.repeatCount = 0
		err = o.PerformOp()
		if err != nil {
			// errors already set and value already put
			return int(o.Size), err
		}
	}

	return int(o.Size), nil
}

/* ------------------------------------------------------------------------------
								Int Engine
--------------------------------------------------------------------------------*/
// IntEngineParams is a struct of parameters for the IntEngine.
type IntEngineParams struct {
	BaseNumericEngineParams
	MinValue  int64 `json:"min"`  // Minimal value of the domain
	MaxValue  int64 `json:"max"`  // Maximal value of the domain
	InitValue int64 `json:"init"` // Initial value in the generator, default = min value.
}

// IntEngine is a field engine which is responsible to generate variables of int types.
// These types can be of lengths 1, 2, 4, 8 bytes (int8, int16, int32, int64).
// The next variable can be generated through different operations, a increment of the current value,
// a decrement of the current value, or some random generation.
type IntEngine struct {
	*IntEngineParams                     // Pointer to params as provided by the caller
	currValue        int64               // Current value in the generator
	domainLen        uint64              // Domain length
	repeatCount      uint16              // Number of times current value was repeated
	mgr              *FieldEngineManager // Field engine manager
}

// CreateIntEngine creates a new FieldEngineIF interface of type IntEngine.
func CreateIntEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// Parse the params
	p := IntEngineParams{InitValue: math.MinInt64}
	p.Step = 1
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create and return new engine
	return NewIntEngine(&p, mgr)
}

// NewIntEngine creates a new IntEngine.
func NewIntEngine(params *IntEngineParams, mgr *FieldEngineManager) (*IntEngine, error) {
	o := new(IntEngine)
	o.mgr = mgr
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.IntEngineParams = params
	o.domainLen = calcDomainLenInt(o.MinValue, o.MaxValue)
	if o.Step > o.domainLen {
		o.Step = o.Step % o.domainLen
	}
	o.currValue = maxInt64(o.MinValue, o.InitValue)
	return o, nil
}

// ValidateParams validates the parameters of the IntEngine.
func (o *IntEngine) validateParams(params *IntEngineParams) (err error) {
	err = nil
	if params.MinValue > params.MaxValue {
		err = fmt.Errorf("Min value %v is bigger than max value %v.", params.MinValue, params.MaxValue)
		o.mgr.counters.maxSmallerThanMin++
	}
	if params.InitValue != math.MinInt64 && (params.InitValue < params.MinValue || params.InitValue > params.MaxValue) {
		err = fmt.Errorf("Init value %v must be between [%v - %v].", params.InitValue, params.MinValue, params.MaxValue)
		o.mgr.counters.badInitValue++
	}
	sizes := []uint16{1, 2, 4, 8}
	maxPossible := []int64{math.MaxInt8, math.MaxInt16, math.MaxInt32, math.MaxInt64}
	minPossible := []int64{math.MinInt8, math.MinInt16, math.MinInt32, math.MinInt64}
	i, ok := findValue(sizes, params.Size)
	if !ok {
		err = fmt.Errorf("Invalid size %v. Size should be {1, 2, 4, 8}.", params.Size)
		o.mgr.counters.invalidSize++
	} else {
		if params.MaxValue > maxPossible[i] {
			err = fmt.Errorf("Max value %v cannot be represented with size %v.", params.MaxValue, params.Size)
			o.mgr.counters.sizeTooSmall++
		}
		if params.MinValue < minPossible[i] {
			err = fmt.Errorf("Min value %v cannot be represented with size %v.", params.MinValue, params.Size)
			o.mgr.counters.sizeTooSmall++
		}
	}
	if params.Op != "inc" && params.Op != "dec" && params.Op != "rand" {
		err = fmt.Errorf("Unsupported operation %v.", params.Op)
		o.mgr.counters.badOperation++
	}
	return err
}

// IncValue increments the value according to step
func (o *IntEngine) IncValue() {
	// Need to be very careful here with overflows.
	left := uint64(o.MaxValue - o.currValue) // this will never overflow as currValue < maxValue
	if o.Step <= left {
		// simple increment by step, not overflow of domain
		// step is fixed module size of domain
		o.currValue += int64(o.Step)
	} else {
		// overflow of domain
		// if here then (step > left) therefore step - left - 1 will not overflow
		o.currValue = o.MinValue + int64(o.Step-left-1) // restart also consumes 1
	}
}

// DecValue decrements the value according to step
func (o *IntEngine) DecValue() {
	left := uint64(o.currValue - o.MinValue) // this will never overflow as currValue > minValue
	if o.Step <= left {
		// no overflow of domain
		// step is fixed module size of domain
		o.currValue -= int64(o.Step)
	} else {
		// overflow of domain
		// if here then (step > left) therefore step - left - 1 will not overflow
		o.currValue = o.MaxValue - int64(o.Step-left-1) // restart also consumes 1
	}
}

// RandValue generates a random value in the domain [min, max]
func (o *IntEngine) RandValue() {
	genValue := rand.Uint64()
	// Converting the value to int64 will give a random int64.
	o.currValue = o.MinValue + int64((genValue % o.domainLen))
}

// PerformOp performs the operation, either it is rand, inc or dec.
func (o *IntEngine) PerformOp() (err error) {
	err = nil
	switch o.Op {
	case "inc":
		o.IncValue()
	case "dec":
		o.DecValue()
	case "rand":
		o.RandValue()
	default:
		o.mgr.counters.badOperation++
		err = errors.New("Unrecognized operation")
	}
	return err
}

// Update implements the Update function of FieldEngineIF.
func (o *IntEngine) Update(b []byte) (int, error) {

	// Converting to uint64 doesn't change the encoding.
	err := PutValue(o.Size, uint64(o.currValue), b, o.mgr)
	if err != nil {
		// errors already set
		return 0, err
	}

	o.repeatCount += 1
	if o.repeatCount >= o.Repeat {
		o.repeatCount = 0
		err = o.PerformOp()
		if err != nil {
			// errors already set and value already put
			return int(o.Size), err
		}
	}

	return int(o.Size), nil
}

/* ------------------------------------------------------------------------------
								FloatEngine
--------------------------------------------------------------------------------*/
// FloatEngineParams is a struct of parameters for the FloatEngine.
type FloatEngineParams struct {
	Size   uint16  `json:"size" validate:"required,oneof=4 8"` // Size can be 4 (float32) or 8 (float64)
	Offset uint16  `json:"offset"`                             // Offset in which to write in the packet
	Min    float64 `json:"min" validate:"required"`            // Minimal value of the domain
	Max    float64 `json:"max" validate:"required"`            // Maximal value of the domain
}

// FloatEngine represents a float32 or float64 engine. For float engines only rand makes sense.
type FloatEngine struct {
	FloatEngineParams                     // Params as provided by the caller
	mgr               *FieldEngineManager // Field engine manager
}

// Update implements the Update function of FieldEngineIF.
func (o *FloatEngine) Update(b []byte) (int, error) {
	if len(b) < int(o.Size) {
		return -1, fmt.Errorf("Size of provided buffer too small, want %v, have %v.", o.Size, len(b))
	}
	factor := rand.Float64()
	genValue := (o.Min) + (o.Max-o.Min)*factor
	if o.Size == 4 {
		binary.BigEndian.PutUint32(b, math.Float32bits(float32(genValue)))
	} else if o.Size == 8 {
		binary.BigEndian.PutUint64(b, math.Float64bits(genValue))
	}
	return int(o.Size), nil
}

// GetOffset implements the GetOffset function of FieldEngineIF.
func (o *FloatEngine) GetOffset() uint16 {
	return o.Offset
}

// GetSize implements the GetSize function of FieldEngineIF.
func (o *FloatEngine) GetSize() uint16 {
	return o.Size
}

// CreateFloatEngine creates a new FieldEngineIF interface of type FloatEngine.
func CreateFloatEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// Parse the params
	p := FloatEngineParams{}
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	if p.Min > p.Max {
		err = fmt.Errorf("Min value %v is bigger than max value %v.", p.Min, p.Max)
		mgr.counters.maxSmallerThanMin++
	}
	// create and return new engine
	return NewFloatEngine(&p, mgr)
}

// NewFloatEngine creates a new FloatEngine.
func NewFloatEngine(params *FloatEngineParams, mgr *FieldEngineManager) (*FloatEngine, error) {
	o := new(FloatEngine)
	o.FloatEngineParams = *params
	o.mgr = mgr
	return o, nil
}

/* ------------------------------------------------------------------------------
								Base List Engine
--------------------------------------------------------------------------------*/

// BaseListEngineParams is a basic struct that gathers the common params for list type engines.
type BaseListEngineParams struct {
	Size      uint16 `json:"size"`       // Size of the uint variable in bytes
	Offset    uint16 `json:"offset"`     // Offset in which to write in the packet
	Op        string `json:"op"`         // Operation which provides the generation, can be {inc, dec, rand}
	Step      uint64 `json:"step"`       // Step to decrement or increment, rand will be ignored. Default=1.
	InitIndex uint64 `json:"init_index"` // Initial index in the list. Default=0.
}

// BaseListEngine is a basic struct that offers common functionality for list type engines.
type BaseListEngine struct {
	*BaseListEngineParams                     // Pointer to params as provided by the caller
	currIndex             int                 // Current index in the generator
	listLength            int                 // Length of the list
	mgr                   *FieldEngineManager // Field engine manager
}

// ValidateParams validates the parameters of the base list engine.
func (o *BaseListEngine) validateParams(params *BaseListEngineParams) (err error) {
	if params.Op != "inc" && params.Op != "dec" && params.Op != "rand" {
		err = fmt.Errorf("Unsupported operation %v.", params.Op)
		o.mgr.counters.badOperation++
	}
	if o.listLength == 0 {
		err = fmt.Errorf("Engine list can't be empty.")
		o.mgr.counters.emptyList++
	}
	if o.listLength > 0 && int(params.InitIndex) >= o.listLength {
		err = fmt.Errorf("Init index %v must be between [%v - %v].", params.InitIndex, 0, o.listLength)
		o.mgr.counters.badInitValue++
	}
	return err
}

// IncValue increments the value according to step
func (o *BaseListEngine) IncValue() {
	o.currIndex = (o.currIndex + int(o.Step)) % o.listLength
}

// DecValue decrements the value according to step
func (o *BaseListEngine) DecValue() {
	// Careful with overflows
	if o.currIndex < int(o.Step) {
		// Overflows
		o.currIndex = o.listLength - (int(o.Step) - o.currIndex)
	} else {
		// No overflow
		o.currIndex = o.currIndex - int(o.Step)
	}
}

// RandValue generates a new index in the list.
func (o *BaseListEngine) RandValue() {
	o.currIndex = rand.Intn(o.listLength)
}

// PerformOp performs the operation, either it is rand, inc or dec.
func (o *BaseListEngine) PerformOp() (err error) {
	err = nil
	switch o.Op {
	case "inc":
		o.IncValue()
	case "dec":
		o.DecValue()
	case "rand":
		o.RandValue()
	default:
		o.mgr.counters.badOperation++
		err = errors.New("Unrecognized operation")
	}
	return err
}

// GetOffset implements the GetOffset function of FieldEngineIF.
func (o *BaseListEngine) GetOffset() uint16 {
	return o.Offset
}

// GetSize implements the GetSize function of FieldEngineIF.
func (o *BaseListEngine) GetSize() uint16 {
	return o.Size
}

/* ------------------------------------------------------------------------------
								UInt List Engine
--------------------------------------------------------------------------------*/
// UIntListEngineParams is a struct of parameters for the UIntListEngine.
type UIntListEngineParams struct {
	BaseListEngineParams          // Embed the basic list params
	List                 []uint64 `json:"list"` // List of choices from which we pick a value and generate
}

// UIntListEngine is a field engine which is responsible to generate variables of uint types.
// These types can be of lengths 1, 2, 4, 8 bytes (uint8, uint16, uint32, uint64).
// The next variable will be picked from the list and can be picked through different operations, an increment of the current index,
// a decrement of the current index, or some random index
type UIntListEngine struct {
	BaseListEngine                       // Embed the common functionality provided by the base class
	uintParams     *UIntListEngineParams // Pointer to params as provided by the caller.
}

// CreateUIntListEngine creates a new FieldEngineIF interface of type UIntListEngine.
func CreateUIntListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// Parse the params.
	p := UIntListEngineParams{}
	p.Step = 1
	p.InitIndex = 0
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	// create and return new engine
	return NewUIntListEngine(&p, mgr)
}

// NewUIntListEngine creates a new uint list engine.
func NewUIntListEngine(params *UIntListEngineParams, mgr *FieldEngineManager) (*UIntListEngine, error) {
	o := new(UIntListEngine)
	o.mgr = mgr
	o.listLength = len(params.List) // set this before validation, it is used in base functionality.
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.uintParams = params
	o.BaseListEngineParams = &params.BaseListEngineParams

	if int(o.Step) > o.listLength {
		o.Step = o.Step % uint64(o.listLength)
	}
	o.currIndex = int(o.InitIndex)
	return o, nil
}

// ValidateParams validates the parameters of the uint engine.
func (o *UIntListEngine) validateParams(params *UIntListEngineParams) (err error) {
	err = o.BaseListEngine.validateParams(&params.BaseListEngineParams)
	sizes := []uint16{1, 2, 4, 8}
	maxPossible := []uint64{math.MaxUint8, math.MaxUint16, math.MaxUint32, math.MaxUint64}
	i, ok := findValue(sizes, params.Size)
	if !ok {
		err = fmt.Errorf("Invalid size %v. Size should be {1, 2, 4, 8}.", params.Size)
		o.mgr.counters.invalidSize++
	} else {
		for j := range params.List {
			if params.List[j] > maxPossible[i] {
				err = fmt.Errorf("List value %v cannot be represented with size %v.", params.List[j], params.Size)
				o.mgr.counters.sizeTooSmall++
				break
			}
		}
	}
	return err
}

// Update implements the Update function of FieldEngineIF.
func (o *UIntListEngine) Update(b []byte) (int, error) {
	err := PutValue(o.Size, o.uintParams.List[o.currIndex], b, o.mgr)
	if err != nil {
		// errors already set
		return 0, err
	}

	err = o.PerformOp()
	if err != nil {
		// errors already set and value already put
		return int(o.Size), err
	}
	return int(o.Size), nil
}

/* ------------------------------------------------------------------------------
								Int List Engine
--------------------------------------------------------------------------------*/
// IntListEngineParams is a struct of parameters for the IntListEngine.
type IntListEngineParams struct {
	BaseListEngineParams         // Embed the basic list params
	List                 []int64 `json:"list"` // List of choices from which we pick a value and generate
}

// IntListEngine is a field engine which is responsible to generate variables of int types.
// These types can be of lengths 1, 2, 4, 8 bytes (int8, int16, int32, int64).
// The next variable will be picked from the list and can be picked through different operations, an increment of the current index,
// a decrement of the current index, or some random index
type IntListEngine struct {
	BaseListEngine                      // Embed the common functionality in a a base class
	intParams      *IntListEngineParams // Pointer to params as provided by the caller.
}

// CreateIntListEngine creates a new FieldEngineIF interface of type IntListEngine.
func CreateIntListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// Parse the params
	p := IntListEngineParams{}
	p.Step = 1
	p.InitIndex = 0
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	// create and return new engine
	return NewIntListEngine(&p, mgr)
}

// NewIntListEngine creates a new uint list engine.
func NewIntListEngine(params *IntListEngineParams, mgr *FieldEngineManager) (*IntListEngine, error) {
	o := new(IntListEngine)
	o.mgr = mgr
	o.listLength = len(params.List) // set this before validation, it is used in base functionality.
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.intParams = params
	o.BaseListEngineParams = &params.BaseListEngineParams

	if int(o.Step) > o.listLength {
		o.Step = o.Step % uint64(o.listLength)
	}
	o.currIndex = int(o.InitIndex)
	return o, nil
}

// ValidateParams validates the parameters of the int engine.
func (o *IntListEngine) validateParams(params *IntListEngineParams) (err error) {
	err = o.BaseListEngine.validateParams(&params.BaseListEngineParams)
	sizes := []uint16{1, 2, 4, 8}
	maxPossible := []int64{math.MaxInt8, math.MaxInt16, math.MaxInt32, math.MaxInt64}
	minPossible := []int64{math.MinInt8, math.MinInt16, math.MinInt32, math.MinInt64}
	i, ok := findValue(sizes, params.Size)
	if !ok {
		err = fmt.Errorf("Invalid size %v. Size should be {1, 2, 4, 8}.", params.Size)
		o.mgr.counters.invalidSize++
	} else {
		for j := range params.List {
			if params.List[j] > maxPossible[i] || params.List[j] < minPossible[i] {
				err = fmt.Errorf("List value %v cannot be represented with size %v.", params.List[j], params.Size)
				o.mgr.counters.sizeTooSmall++
				break
			}
		}
	}
	return err
}

// Update implements the Update function of FieldEngineIF.
func (o *IntListEngine) Update(b []byte) (int, error) {
	// Converting to uint64 doesn't change the value or the binary encoding, only the way
	// the value is interpreted.
	err := PutValue(o.Size, uint64(o.intParams.List[o.currIndex]), b, o.mgr)
	if err != nil {
		// errors already set
		return 0, err
	}

	err = o.PerformOp()
	if err != nil {
		// errors already set and value already put
		return int(o.Size), err
	}
	return int(o.Size), nil
}

/* ------------------------------------------------------------------------------
								Float List Engine
--------------------------------------------------------------------------------*/
// FloatListEngineParams is a struct of parameters for the FloatListEngine.
type FloatListEngineParams struct {
	BaseListEngineParams           // Embed the basic list params
	List                 []float64 `json:"list"` // List of choices from which we pick a value and generate
}

// FloatListEngine is a field engine which is responsible to generate variables of float types.
// These types can be of lengths 4, 8 bytes (float32, float64).
// The next variable will be picked from the list and can be picked through different operations, an increment of the current index,
// a decrement of the current index, or some random index
type FloatListEngine struct {
	BaseListEngine                        // Embed the common functionality in a a base class
	floatParams    *FloatListEngineParams // Pointer to params as provided by the caller.
}

// CreateFloatListEngine creates a new FieldEngineIF interface of type FloatListEngine.
func CreateFloatListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// parse the json
	p := FloatListEngineParams{}
	p.Step = 1
	p.InitIndex = 0
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	// create and return new engine
	return NewFloatListEngine(&p, mgr)
}

// NewFloatListEngine creates a new uint list engine.
func NewFloatListEngine(params *FloatListEngineParams, mgr *FieldEngineManager) (*FloatListEngine, error) {
	o := new(FloatListEngine)
	o.mgr = mgr
	o.listLength = len(params.List) // set this before validation, it is used in base functionality.
	err := o.BaseListEngine.validateParams(&params.BaseListEngineParams)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.floatParams = params
	o.BaseListEngineParams = &params.BaseListEngineParams

	if int(o.Step) > o.listLength {
		o.Step = o.Step % uint64(o.listLength)
	}
	o.currIndex = int(o.InitIndex)
	return o, nil
}

// Update implements the Update function of FieldEngineIF.
func (o *FloatListEngine) Update(b []byte) (int, error) {
	// Converting to uint64 doesn't change the value or the binary encoding, only the way
	// the value is interpreted.
	var val uint64
	if o.Size == 4 {
		val = uint64(math.Float32bits(float32(o.floatParams.List[o.currIndex])))
	} else if o.Size == 8 {
		val = math.Float64bits(o.floatParams.List[o.currIndex])
	}
	err := PutValue(o.Size, val, b, o.mgr)
	if err != nil {
		// errors already set
		return 0, err
	}

	err = o.PerformOp()
	if err != nil {
		// errors already set and value already put
		return int(o.Size), err
	}
	return int(o.Size), nil
}

/* ------------------------------------------------------------------------------
								String List Engine
--------------------------------------------------------------------------------*/
// StringListEngineParams is a struct of parameters for the StringListEngine.
type StringListEngineParams struct {
	BaseListEngineParams
	List         []string `json:"list"`          // List of choices from which we pick a value and generate
	PaddingValue uint8    `json:"padding_value"` // Value which will be used to pad the string to maximal size
}

// StringListEngine is a field engine which is responsible to generate variables of string types.
// The strings can be any length as long as they are shorter than size. They will be padded with the padding_value to size length.
// The next variable will be picked from the list and can be picked through different operations, an increment of the current index,
// a decrement of the current index, or some random index
type StringListEngine struct {
	BaseListEngine                         // Embed base list engine for base functionality
	stringParams   *StringListEngineParams // Pointer to params as provided by the caller.
	processedList  [][]byte                // List of bytes manipulated and padded ahead of time for fast update
}

// CreateStringListEngine creates a new FieldEngineIF interface of type StringListEngine.
func CreateStringListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// Parse the params
	p := StringListEngineParams{PaddingValue: 0}
	p.Step = 1
	p.InitIndex = 0
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	// create and return new engine
	return NewStringListEngine(&p, mgr)
}

// NewStringListEngine creates a new string list engine.
func NewStringListEngine(params *StringListEngineParams, mgr *FieldEngineManager) (*StringListEngine, error) {
	o := new(StringListEngine)
	o.mgr = mgr
	o.listLength = len(params.List) // set this before validation, it is used in base functionality.
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.stringParams = params
	o.BaseListEngineParams = &params.BaseListEngineParams
	if int(o.Step) > o.listLength {
		o.Step = o.Step % uint64(o.listLength)
	}
	o.processStrings()
	o.currIndex = int(o.InitIndex)
	return o, nil
}

// ValidateParams validates the parameters of the string list engine.
func (o *StringListEngine) validateParams(params *StringListEngineParams) (err error) {
	err = o.BaseListEngine.validateParams(&params.BaseListEngineParams)
	for _, str := range params.List {
		if len([]byte(str)) > int(params.Size) {
			err = fmt.Errorf("String %v cannot be represented with size %v.", str, params.Size)
			o.mgr.counters.sizeTooSmall++
			break
		}
	}
	return err
}

// processStrings pre processes the strings and paddes them ahead of time in order to perform fast update
// and no need to pad every time.
func (o *StringListEngine) processStrings() {
	for _, str := range o.stringParams.List {
		bytesStr := []byte(str)
		appendSize := int(o.Size) - len(bytesStr)
		for j := 0; j < appendSize; j++ {
			bytesStr = append(bytesStr, byte(o.stringParams.PaddingValue))
		}
		o.processedList = append(o.processedList, bytesStr)
	}
}

// Update implements the Update function of FieldEngineIF.
// This will always pad and return the padded size.
func (o *StringListEngine) Update(b []byte) (int, error) {
	if len(b) < int(o.Size) {
		o.mgr.counters.bufferTooShort++
		return 0, fmt.Errorf("Provided slice is shorter that the size of the variable to write, want at least %v, have %v.", o.Size, len(b))
	}

	copied := copy(b, o.processedList[o.currIndex])
	if copied != int(o.Size) {
		o.mgr.counters.badCopyToBuffer++
		return 0, fmt.Errorf("Error happened copying the string into the buffer. Copied bytes: want %v and have %v.", o.Size, copied)
	}

	err := o.PerformOp()
	if err != nil {
		// errors already set and value already put
		return int(o.Size), err
	}
	return int(o.Size), nil
}

/* ------------------------------------------------------------------------------
							HistogramEngine
--------------------------------------------------------------------------------*/
// HistogramEngineCommonParams contains the common params for all the types of
// histogram engines.
type HistogramEngineCommonParams struct {
	Size   uint16 `json:"size"`   // Size of the variable in bytes
	Offset uint16 `json:"offset"` // Offset in which to write in the packet
}

// HistogramEngineParams is a struct that must be provided to the HistogramEngine
// when creating it.
type HistogramEngineParams struct {
	HistogramEngineCommonParams                  // Common params for all the engines
	Entries                     []HistogramEntry // The entries of the histogram
}

// HistogramEngine is a FieldEngine, which contains a non uniform pseudo random
// generator and can generate it's entries as per probability of each entry.
type HistogramEngine struct {
	par           *HistogramEngineParams // Pointer to params as provided by the caller.
	distributions []uint32               // Distribution slice
	generator     *NonUniformRandGen     // Non uniform random generator of the distribution
	mgr           *FieldEngineManager    // Field engine manager

}

// NewHistogramEngine creates a new HistogramEngine from the HistogramEngineParams provided.
func NewHistogramEngine(params *HistogramEngineParams, mgr *FieldEngineManager) (o *HistogramEngine, err error) {
	o = new(HistogramEngine)
	o.buildDistributionSlice(params.Entries)
	o.par = params
	o.mgr = mgr
	o.generator, err = NewNonUniformRandGen(o.distributions)
	if err != nil {
		o.mgr.counters.generatorCreationError++
		return nil, err
	}
	return o, nil
}

func (o *HistogramEngine) buildDistributionSlice(entries []HistogramEntry) {
	for _, entry := range entries {
		o.distributions = append(o.distributions, entry.GetProb())
	}
}

// Update implements the Update function of FieldEngineIF.
func (o *HistogramEngine) Update(b []byte) (int, error) {
	if len(b) < int(o.par.Size) {
		o.mgr.counters.bufferTooShort++
		return 0, fmt.Errorf("Provided slice is shorter that the size of the variable to write, want at least %v, have %v.", o.par.Size, len(b))
	}
	// clean the provided buffer
	copiedSize := copy(b[:o.par.Size], make([]byte, o.par.Size))
	if copiedSize != int(o.par.Size) {
		o.mgr.counters.badCopyToBuffer++
		return 0, fmt.Errorf("Didn't copy the right amount to the buffer, want %v have %v.", o.par.Size, copiedSize)
	}
	entryIndex := o.generator.Generate()
	entry := o.par.Entries[entryIndex]
	newValueBytes, err := entry.GetValue(o.par.Size)
	if err != nil {
		o.mgr.counters.invalidHistogramEntry++
		return 0, err
	}
	if len(newValueBytes) > int(o.par.Size) {
		o.mgr.counters.sizeTooSmall++
		return 0, fmt.Errorf("Size %v is too small for generated value with length %v.", o.par.Size, len(newValueBytes))
	}
	copiedSize = copy(b[:o.par.Size], newValueBytes)
	if copiedSize != len(newValueBytes) {
		o.mgr.counters.badCopyToBuffer++
		return 0, fmt.Errorf("Didn't copy the right amount to the buffer, want %v have %v.", len(newValueBytes), copiedSize)
	}
	return copiedSize, nil
}

// GetOffset implements the GetOffset function of FieldEngineIF.
func (o *HistogramEngine) GetOffset() uint16 {
	return o.par.Offset
}

// GetSize implements the GetSize function of FieldEngineIF.
func (o *HistogramEngine) GetSize() uint16 {
	return o.par.Size
}

/* ------------------------------------------------------------------------------
						Helping Utils for Uint32 histogram
--------------------------------------------------------------------------------*/

func putValueInBuffer(size uint16, value uint32) (b []byte, err error) {
	b = make([]byte, size)
	switch size {
	case 1:
		b[0] = uint8(value)
	case 2:
		binary.BigEndian.PutUint16(b, uint16(value))
	case 4:
		binary.BigEndian.PutUint32(b, value)
	default:
		return nil, fmt.Errorf("Invalid size %v in GetValue!", size)
	}
	return b, nil
}

/* ------------------------------------------------------------------------------
						HistogramUInt32Entry
--------------------------------------------------------------------------------*/
// HistogramUInt32Entry represents a uint32 which can be used as an entry for the
// HistogramEngine. This entry can be picked with probability prob.
type HistogramUInt32Entry struct {
	V    uint32 `json:"v"`    // A value v of 32 bits
	Prob uint32 `json:"prob"` // Probability of this entry
}

// HistogramUInt32Params is used for parsing the input json.
type HistogramUInt32Params struct {
	HistogramEngineCommonParams                        // Common params
	Entries                     []HistogramUInt32Entry `json:"entries"` // slice of entries
}

// GetValue puts the value in the byte buffer.
func (o *HistogramUInt32Entry) GetValue(size uint16) (b []byte, err error) {
	return putValueInBuffer(size, o.V)
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramUInt32Entry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt32ListEngine creates an histogram engine of uint 32 from the input json.
func CreateHistogramUInt32Engine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramUInt32Params{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		switch p.Size {
		case 1:
			if p.Entries[i].V > 0xFF {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Value %v can't be represented with %v bytes\n.", p.Entries[i].V, p.Size)
			}
		case 2:
			if p.Entries[i].V > 0xFFFF {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Value %v can't be represented with %v bytes\n.", p.Entries[i].V, p.Size)
			}
		case 4:
		default:
			mgr.counters.invalidSize++
			return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramInt32Entry
--------------------------------------------------------------------------------*/
// HistogramInt32Entry represents a int32 which can be used as an entry for the
// HistogramEngine. This entry can be picked with probability prob.
type HistogramInt32Entry struct {
	V    int32  `json:"v"`    // A value v of 32 bits
	Prob uint32 `json:"prob"` // Probability of this entry
}

// HistogramInt32Params is used for parsing the input json.
type HistogramInt32Params struct {
	HistogramEngineCommonParams                       // Common params
	Entries                     []HistogramInt32Entry `json:"entries"` // Slice of entries
}

// GetValue puts the value in the byte buffer.
func (o *HistogramInt32Entry) GetValue(size uint16) (b []byte, err error) {
	// Converting to uint32 won't change the value, only the representation.
	return putValueInBuffer(size, uint32(o.V))
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramInt32Entry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt32ListEngine creates an histogram engine of int32 from the input json.
func CreateHistogramInt32Engine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramInt32Params{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		switch p.Size {
		case 1:
			if p.Entries[i].V > math.MaxInt8 || p.Entries[i].V < math.MinInt8 {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Value %v can't be represented with %v bytes\n.", p.Entries[i].V, p.Size)
			}
		case 2:
			if p.Entries[i].V > math.MaxInt16 || p.Entries[i].V < math.MinInt16 {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Value %v can't be represented with %v bytes\n.", p.Entries[i].V, p.Size)
			}
		case 4:
		default:
			mgr.counters.invalidSize++
			return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramUInt32RangeEntry
--------------------------------------------------------------------------------*/
// HistogramUInt32RangeEntry represents a range of uint32 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the range will be generated uniformly.
type HistogramUInt32RangeEntry struct {
	Min  uint32 `json:"min"`  // Lower bound of the range
	Max  uint32 `json:"max"`  // Higher bound of the range
	Prob uint32 `json:"prob"` // Probability of this entry
}

// HistogramUInt32RangeParams is used for parsing the input json.
type HistogramUInt32RangeParams struct {
	HistogramEngineCommonParams                             // Common params
	Entries                     []HistogramUInt32RangeEntry `json:"entries"` // Slice of entries
}

// GetValue generates uniformly a value in the range and puts it in the byte buffer.
func (o *HistogramUInt32RangeEntry) GetValue(size uint16) (b []byte, err error) {
	if o.Max < o.Min {
		return nil, fmt.Errorf("Max %v is smaller than min %v in HistogramRuneRangeEntry.", o.Max, o.Min)
	}
	v := rand.Uint32()                    // generate random 32 bytes
	v = o.Min + (v % (o.Max - o.Min + 1)) // scale it to the domain
	return putValueInBuffer(size, v)
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramUInt32RangeEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt32RangeEngine creates an histogram engine of uint32 range from the input json.
func CreateHistogramUInt32RangeEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramUInt32RangeParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if p.Entries[i].Min > p.Entries[i].Max {
			mgr.counters.maxSmallerThanMin++
			return nil, fmt.Errorf("Min %v bigger than max %v in entry #%v.", p.Entries[i].Min, p.Entries[i].Max, i)
		}
		switch p.Size {
		case 1:
			if p.Entries[i].Max > 0xFF {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Max %v can't be represented with %v bytes.", p.Entries[i].Max, p.Size)
			}
		case 2:
			if p.Entries[i].Max > 0xFFFF {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Max %v can't be represented with %v bytes.", p.Entries[i].Max, p.Size)
			}
		case 4:
		default:
			mgr.counters.invalidSize++
			return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
		}

		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramInt32RangeEntry
--------------------------------------------------------------------------------*/
// HistogramInt32RangeEntry represents a range of int32 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the range will be generated uniformly.
type HistogramInt32RangeEntry struct {
	Min          int32  `json:"min"`  // Lower bound of the range
	Max          int32  `json:"max"`  // Higher bound of the range
	Prob         uint32 `json:"prob"` // Probability of this entry
	domainLength uint64 // Domain Length, (max - min + 1)
}

// HistogramInt32RangeParams is used for parsing the input json.
type HistogramInt32RangeParams struct {
	HistogramEngineCommonParams                            // Common params
	Entries                     []HistogramInt32RangeEntry `json:"entries"` // Slice of entries
}

// GetValue generates uniformly a value in the range and puts it in the byte buffer.
func (o *HistogramInt32RangeEntry) GetValue(size uint16) (b []byte, err error) {
	genValue := rand.Uint64()                   // generate random 32 bytes
	v := o.Min + int32(genValue%o.domainLength) // scale it to the domain
	return putValueInBuffer(size, uint32(v))
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramInt32RangeEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramInt32RangeEngine creates an histogram engine of int32 range from the input json.
func CreateHistogramInt32RangeEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramInt32RangeParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if p.Entries[i].Min > p.Entries[i].Max {
			mgr.counters.maxSmallerThanMin++
			return nil, fmt.Errorf("Min %v bigger than max %v in entry #%v.", p.Entries[i].Min, p.Entries[i].Max, i)
		}
		switch p.Size {
		case 1:
			if p.Entries[i].Max > math.MaxInt8 || p.Entries[i].Min < math.MinInt8 {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Range [%v-%v] can't be represented with %v bytes\n.", p.Entries[i].Min, p.Entries[i].Max, p.Size)
			}
		case 2:
			if p.Entries[i].Max > math.MaxInt16 || p.Entries[i].Min < math.MinInt16 {
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Range [%v-%v] can't be represented with %v bytes\n.", p.Entries[i].Min, p.Entries[i].Max, p.Size)
			}
		case 4:
		default:
			mgr.counters.invalidSize++
			return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
		}
		p.Entries[i].domainLength = calcDomainLenInt(int64(p.Entries[i].Min), int64(p.Entries[i].Max))
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramUInt32ListEntry
--------------------------------------------------------------------------------*/
// HistogramUInt32ListEntry represents a list of uint32 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the list will be selected uniformly.
type HistogramUInt32ListEntry struct {
	List []uint32 `json:"list"` // A list from where the element will be picked
	Prob uint32   `json:"prob"` // Probability of this entry
}

// HistogramUInt32ListParams is used for parsing the input json.
type HistogramUInt32ListParams struct {
	HistogramEngineCommonParams                            // common params
	Entries                     []HistogramUInt32ListEntry `json:"entries"` // slice of entries
}

// GetValue picks a random value from the list and puts it in the byte buffer.
func (o *HistogramUInt32ListEntry) GetValue(size uint16) (b []byte, err error) {
	if o.List == nil || len(o.List) == 0 {
		return nil, fmt.Errorf("Empty list in HistogramUInt32ListEntry.")
	}
	index := rand.Intn(len(o.List))
	return putValueInBuffer(size, o.List[index])
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramUInt32ListEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt32ListEngine creates an histogram engine of uint32 list from the input json.
func CreateHistogramUInt32ListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramUInt32ListParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if len(p.Entries[i].List) == 0 {
			mgr.counters.emptyList++
			return nil, fmt.Errorf("Entry # %v contains an empty list.", i)
		}
		for j := range p.Entries[i].List {
			v := p.Entries[i].List[j]
			switch p.Size {
			case 1:
				if v > 0xFF {
					mgr.counters.invalidSize++
					return nil, fmt.Errorf("List value %v can't be represented with %v bytes\n.", v, p.Size)
				}
			case 2:
				if v > 0xFFFF {
					mgr.counters.invalidSize++
					return nil, fmt.Errorf("List value %v can't be represented with %v bytes\n.", v, p.Size)
				}
			case 4:
			default:
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
			}
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramInt32ListEntry
--------------------------------------------------------------------------------*/
// HistogramInt32ListEntry represents a list of int32 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the list will be selected uniformly.
type HistogramInt32ListEntry struct {
	List []int32 `json:"list"` // A list from where the element will be picked
	Prob uint32  `json:"prob"` // Probability of this entry
}

// HistogramInt32ListParams is used for parsing the input json.
type HistogramInt32ListParams struct {
	HistogramEngineCommonParams                           // Common params
	Entries                     []HistogramInt32ListEntry `json:"entries"` // Slice of entries
}

// GetValue picks a random value from the list and puts it in the byte buffer.
func (o *HistogramInt32ListEntry) GetValue(size uint16) (b []byte, err error) {
	if o.List == nil || len(o.List) == 0 {
		return nil, fmt.Errorf("Empty list in HistogramUInt32ListEntry.")
	}
	index := rand.Intn(len(o.List))
	return putValueInBuffer(size, uint32(o.List[index]))
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramInt32ListEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramInt32ListEngine creates an histogram engine of int32 list from the input json.
func CreateHistogramInt32ListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramInt32ListParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if len(p.Entries[i].List) == 0 {
			mgr.counters.emptyList++
			return nil, fmt.Errorf("Entry # %v contains an empty list.", i)
		}
		for j := range p.Entries[i].List {
			v := p.Entries[i].List[j]
			switch p.Size {
			case 1:
				if v > math.MaxInt8 || v < math.MinInt8 {
					mgr.counters.invalidSize++
					return nil, fmt.Errorf("List value %v can't be represented with %v bytes\n.", v, p.Size)
				}
			case 2:
				if v > math.MaxInt16 || v < math.MinInt16 {
					mgr.counters.invalidSize++
					return nil, fmt.Errorf("List value %v can't be represented with %v bytes\n.", v, p.Size)
				}
			case 4:
			default:
				mgr.counters.invalidSize++
				return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
			}
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramUInt64Entry
--------------------------------------------------------------------------------*/
// HistogramUInt64Entry represents a uint64 which can be used as an entry for the
// HistogramEngine. This entry can be picked with probability prob.
type HistogramUInt64Entry struct {
	V    uint64 `json:"v"`    // A value v of 64 bits
	Prob uint32 `json:"prob"` // Probability of this entry
}

// HistogramUInt64Params is used for parsing the input json.
type HistogramUInt64Params struct {
	HistogramEngineCommonParams                        // Common params
	Entries                     []HistogramUInt64Entry `json:"entries"` // Slice of entries
}

// GetValue puts the value in the byte buffer.
func (o *HistogramUInt64Entry) GetValue(size uint16) (b []byte, err error) {
	if size != 8 {
		return nil, fmt.Errorf("Size in HistogramUInt64Entry GetValue is %v, want %v.", size, 8)
	}
	b = make([]byte, size)
	binary.BigEndian.PutUint64(b, o.V)
	return b, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramUInt64Entry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt64Engine creates an histogram engine of uint64 from the input json.
func CreateHistogramUInt64Engine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramUInt64Params{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	if p.Size != 8 {
		mgr.counters.invalidSize++
		return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramInt64Entry
--------------------------------------------------------------------------------*/
// HistogramInt64Entry represents a int64 which can be used as an entry for the
// HistogramEngine. This entry can be picked with probability prob.
type HistogramInt64Entry struct {
	V    int64  `json:"v"`    // A value v of 64 bits
	Prob uint32 `json:"prob"` // Probability of this entry
}

// HistogramInt64Params is used for parsing the input json.
type HistogramInt64Params struct {
	HistogramEngineCommonParams                       // Common params
	Entries                     []HistogramInt64Entry `json:"entries"` // Slice of entries
}

// GetValue puts the value in the byte buffer.
func (o *HistogramInt64Entry) GetValue(size uint16) (b []byte, err error) {
	if size != 8 {
		return nil, fmt.Errorf("Size in HistogramUInt64Entry GetValue is %v, want %v.", size, 8)
	}
	b = make([]byte, size)
	binary.BigEndian.PutUint64(b, uint64(o.V))
	return b, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramInt64Entry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramInt64Engine creates an histogram engine of int64 from the input json.
func CreateHistogramInt64Engine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramInt64Params{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	if p.Size != 8 {
		mgr.counters.invalidSize++
		return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramUInt64RangeEntry
--------------------------------------------------------------------------------*/
// HistogramUInt64RangeEntry represents a range of uint64 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the range will be generated uniformly.
type HistogramUInt64RangeEntry struct {
	Min  uint64 `json:"min"`  // Lower bound of the range
	Max  uint64 `json:"max"`  // Higher bound of the range
	Prob uint32 `json:"prob"` // Probability of this entry
}

// HistogramUInt64RangeParams is used for parsing the input json.
type HistogramUInt64RangeParams struct {
	HistogramEngineCommonParams                             // Common params
	Entries                     []HistogramUInt64RangeEntry `json:"entries"` // Slice of entries
}

// GetValue generates uniformly a value in the range and puts it in the byte buffer.
func (o *HistogramUInt64RangeEntry) GetValue(size uint16) (b []byte, err error) {
	b = make([]byte, size)
	v := rand.Uint64()                    // generate random 64 bytes
	v = o.Min + (v % (o.Max - o.Min + 1)) // scale it to the domain
	binary.BigEndian.PutUint64(b, v)      // put it in the bytes buffer
	return b, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramUInt64RangeEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt64RangeEngine creates an histogram engine of uint64 range from the input json.
func CreateHistogramUInt64RangeEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramUInt64RangeParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	if p.Size != 8 {
		mgr.counters.invalidSize++
		return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if p.Entries[i].Min > p.Entries[i].Max {
			mgr.counters.maxSmallerThanMin++
			return nil, fmt.Errorf("Min %v bigger than max %v in entry #%v.", p.Entries[i].Min, p.Entries[i].Max, i)
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramInt64RangeEntry
--------------------------------------------------------------------------------*/
// HistogramInt64RangeEntry represents a range of int64 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the range will be generated uniformly.
type HistogramInt64RangeEntry struct {
	Min          int64  `json:"min"`  // Lower bound of the range
	Max          int64  `json:"max"`  // Higher bound of the range
	Prob         uint32 `json:"prob"` // Probability of this entry
	domainLength uint64 // Max - Min + 1
}

// HistogramInt64RangeParams is used for parsing the input json.
type HistogramInt64RangeParams struct {
	HistogramEngineCommonParams                            // Common params
	Entries                     []HistogramInt64RangeEntry `json:"entries"` // Slice of entries
}

// GetValue generates uniformly a value in the range and puts it in the byte buffer.
func (o *HistogramInt64RangeEntry) GetValue(size uint16) (b []byte, err error) {
	b = make([]byte, size)
	genValue := rand.Uint64()                   // generate random 64 bytes
	v := o.Min + int64(genValue%o.domainLength) // scale it to the domain
	binary.BigEndian.PutUint64(b, uint64(v))    // put it in the bytes buffer
	return b, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramInt64RangeEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramInt64RangeEngine creates an histogram engine of uint64 range from the input json.
func CreateHistogramInt64RangeEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramInt64RangeParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	if p.Size != 8 {
		mgr.counters.invalidSize++
		return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if p.Entries[i].Min > p.Entries[i].Max {
			mgr.counters.maxSmallerThanMin++
			return nil, fmt.Errorf("Min %v bigger than max %v in entry #%v.", p.Entries[i].Min, p.Entries[i].Max, i)
		}
		p.Entries[i].domainLength = calcDomainLenInt(p.Entries[i].Min, p.Entries[i].Max)
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramUInt64ListEntry
--------------------------------------------------------------------------------*/
// HistogramUInt64ListEntry represents a list of uint64 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the list will be selected uniformly.
type HistogramUInt64ListEntry struct {
	List []uint64 `json:"list"` // A list from where the element will be picked
	Prob uint32   `json:"prob"` // Probability of this entry
}

// HistogramUInt64ListParams is used for parsing the input json.
type HistogramUInt64ListParams struct {
	HistogramEngineCommonParams                            // Common params
	Entries                     []HistogramUInt64ListEntry `json:"entries"` // Slice of entries
}

// GetValue picks a random value from the list and puts it in the byte buffer.
func (o *HistogramUInt64ListEntry) GetValue(size uint16) (b []byte, err error) {
	b = make([]byte, size)
	index := rand.Intn(len(o.List))
	binary.BigEndian.PutUint64(b, o.List[index])
	return b, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramUInt64ListEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramUInt64ListEngine creates an histogram engine of uint64 list from the input json.
func CreateHistogramUInt64ListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramUInt64ListParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	if p.Size != 8 {
		mgr.counters.invalidSize++
		return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if len(p.Entries[i].List) == 0 {
			mgr.counters.emptyList++
			return nil, fmt.Errorf("Entry # %v contains an empty list.", i)
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
						HistogramInt64ListEntry
--------------------------------------------------------------------------------*/
// HistogramInt64ListEntry represents a list of int64 which can be used as an
// entry for the HistogramEngine. This entry can be picked with probability prob.
// If the entry is picked, a value in the list will be selected uniformly.
type HistogramInt64ListEntry struct {
	List []int64 `json:"list"` // A list from where the element will be picked
	Prob uint32  `json:"prob"` // Probability of this entry
}

// HistogramInt64ListParams is used for parsing the input json.
type HistogramInt64ListParams struct {
	HistogramEngineCommonParams                           // Common params
	Entries                     []HistogramInt64ListEntry `json:"entries"` // Slice of entries
}

// GetValue picks a random value from the list and puts it in the byte buffer.
func (o *HistogramInt64ListEntry) GetValue(size uint16) (b []byte, err error) {
	b = make([]byte, size)
	index := rand.Intn(len(o.List))
	binary.BigEndian.PutUint64(b, uint64(o.List[index]))
	return b, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramInt64ListEntry) GetProb() uint32 {
	return o.Prob
}

// CreateHistogramInt64ListEngine creates an histogram engine of int64 list from the input json.
func CreateHistogramInt64ListEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramInt64ListParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	if p.Size != 8 {
		mgr.counters.invalidSize++
		return nil, fmt.Errorf("Invalid size %v with this type of engine.", p.Size)
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		if len(p.Entries[i].List) == 0 {
			mgr.counters.emptyList++
			return nil, fmt.Errorf("Entry # %v contains an empty list.", i)
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
					 	Time Start Engine
--------------------------------------------------------------------------------*/
// TimeStartEngineParams represents the paramaters for a TimeStartEngine engine.
type TimeStartEngineParams struct {
	Size              uint16 `json:"size"`                                     // Size of the time field in bytes. Can be 4 or 8.
	Offset            uint16 `json:"offset"`                                   // Offset in which to write in the packet
	TimeEndEngineName string `json:"time_end_engine_name" validate:"required"` // Name of the time end engine.
	UptimeOffset      uint64 `json:"time_offset"`                              // The first packet offset from the starting point in milliseconds.
	// In case of relative time from systime provide the offset to uptime. In case of absolute of time, provide the Unix time.
	InterPacketGapMin uint64 `json:"ipg_min" validate:"required"` // Minimal Inter Packet Gap of this flow in milliseconds.
	InterPacketGapMax uint64 `json:"ipg_max" validate:"required"` // Maximal Inter Packet Gap of this flow in milliseconds.
	// The next flow will have an Inter Packet Gap between the minimal and maximal IPG from the end of the previous flow.
}

// TimeStartEngine represents an engine for time beginning fields. (relative to uptime for example, or absolute Unix time).
// This engine should be used together with a TimeEndEngine.
type TimeStartEngine struct {
	par             *TimeStartEngineParams // Parameters
	mgr             *FieldEngineManager    // Engine Manager
	newFlowStart    uint64                 // Relative time for this flow to begin.
	previousFlowEnd uint64                 // Relative time since the previous flow ended. This value is updated by the TimeEndEngine.
	firstPacket     bool                   // Flag representing if this is the first packet
	timeEndEngine   *TimeEndEngine         // TimeEndEngine
}

// CreateTimeStartEngine creates a TimeStartEngine.
func CreateTimeStartEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// parse the json
	p := TimeStartEngineParams{UptimeOffset: 0}
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	// create and return new engine
	return NewTimeStartEngine(&p, mgr)
}

// NewTimeStartEngine creates a new TimeStart engine. This engine can be used for fields such as
// flowStartSysUpTime etc.
func NewTimeStartEngine(params *TimeStartEngineParams, mgr *FieldEngineManager) (*TimeStartEngine, error) {
	o := new(TimeStartEngine)
	o.mgr = mgr
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.par = params
	o.newFlowStart = o.par.UptimeOffset
	o.firstPacket = true
	return o, nil
}

// validateParams validates that the parameters are correct.
func (o *TimeStartEngine) validateParams(params *TimeStartEngineParams) (err error) {
	if params.Size != 4 && params.Size != 8 {
		o.mgr.counters.invalidSize++
		err = fmt.Errorf("Size for this engine must be 4 or 8. Invalid size %v.", params.Size)
	}

	if params.InterPacketGapMin > params.InterPacketGapMax {
		o.mgr.counters.maxSmallerThanMin++
		err = fmt.Errorf("InterPacketGap min %v is greater than InterPacketGap max %v.", params.InterPacketGapMin, params.InterPacketGapMax)
	}

	if params.Size == 4 && (params.InterPacketGapMax > 0xFFFFFFFF || params.UptimeOffset > 0xFFFFFFFF) {
		o.mgr.counters.sizeTooSmall++
		err = fmt.Errorf("Size too small, can't represent max value or uptime offset with %v bytes.", params.Size)
	}

	return err
}

// setPreviousFlowEndTime is a setter provided to the TimeEnd engine to set the previous flow timestamp (the one that is about to finishfor End engine.)
func (o *TimeStartEngine) setPreviousFlowEndTime(previousFlowStartTime uint64) {
	o.previousFlowEnd = previousFlowStartTime
}

// getNewFlowStartValue generates a relative timestamp for the beginning of the new flow.
func (o *TimeStartEngine) getNewFlowStartValue() (value uint64) {
	if o.firstPacket {
		value = o.newFlowStart
		o.firstPacket = false
	} else {
		// Generates a uint64 with uniform distribution.
		// Converts the generated value to a value in the domain [min-max]. This way we get an ipg.
		genValue := rand.Uint64()
		ipg := o.par.InterPacketGapMin + (genValue % (o.par.InterPacketGapMax - o.par.InterPacketGapMin + 1))
		// Assumes the TimeEnd engine has updated the previous flow end.
		value = o.previousFlowEnd + ipg
	}
	return value
}

// Update implements the Update function of FieldEngineIF.
func (o *TimeStartEngine) Update(b []byte) (int, error) {

	// Get the Time End engine. This can't be done upon creation as the engine might not be created yet.
	if o.timeEndEngine == nil {
		timeEndEngine, ok := o.mgr.engines[o.par.TimeEndEngineName]
		if !ok {
			return 0, fmt.Errorf("TimeEnd engine name %v not found in engine manager database. Must provide this engine.", o.par.TimeEndEngineName)
		}
		o.timeEndEngine, ok = timeEndEngine.(*TimeEndEngine)
		if !ok {
			o.mgr.counters.badEngineType++
			return 0, fmt.Errorf("Failed converting engine %v to TimeEnd. Make sure the engine type is corrent.", o.par.TimeEndEngineName)
		}
	}

	// Get a value for the new flow start time.
	value := o.getNewFlowStartValue()

	// Update the buffer
	switch o.par.Size {
	case 4:
		binary.BigEndian.PutUint32(b, uint32(value))
	case 8:
		binary.BigEndian.PutUint64(b, value)
	default:
		o.mgr.counters.invalidSize++
		return 0, errors.New("Size should be 4 or 8.")
	}

	// Update the time end engine so it will know to calculate the new value
	o.timeEndEngine.setCurrentFlowStartTime(value)
	return int(o.par.Size), nil
}

// GetOffset implements the GetOffset function of FieldEngineIF.
func (o *TimeStartEngine) GetOffset() uint16 {
	return o.par.Offset
}

// GetSize implements the GetSize function of FieldEngineIF.
func (o *TimeStartEngine) GetSize() uint16 {
	return o.par.Size
}

/* ------------------------------------------------------------------------------
							Time End Engine
--------------------------------------------------------------------------------*/
// TimeEndEngineParams represents the paramaters for a TimeEndEngine.
type TimeEndEngineParams struct {
	Size                uint16 `json:"size"`                                       // Size of the time field in bytes. Can be 4 or 8.
	Offset              uint16 `json:"offset"`                                     // Offset in which to write in the packet
	TimeStartEngineName string `json:"time_start_engine_name" validate:"required"` // Name of the time start engine.
	DurationMin         uint64 `json:"duration_min" validate:"required"`           // Minimal duration of this flow in milliseconds.
	DurationMax         uint64 `json:"duration_max" validate:"required"`           // Maximal duration of this flow in milliseconds.
	// This flow will have a duration between the minimal and maximal duration provided. Value is randomly generated.
}

// TimeEndEngine represents an engine for time end fields. (relative to uptime for example, or absolute Unix time).
// This engine should be used together with a TimeStartEngine.
type TimeEndEngine struct {
	par              *TimeEndEngineParams // Parameters
	mgr              *FieldEngineManager  // Engine Manager
	currentFlowStart uint64               // Relative time since this flow began. This field is updated by the time start engine.
	currentFlowEnd   uint64               // Relative time for this flow to end. This value we generate.
	timeStartEngine  *TimeStartEngine     // Relative Time start engine
}

// CreateTimeEndEngine
func CreateTimeEndEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (FieldEngineIF, error) {
	// parse the json
	p := TimeEndEngineParams{}
	err := mgr.tctx.UnmarshalValidate(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}
	// create and return new engine
	return NewTimeEndEngine(&p, mgr)
}

// NewTimeEndEngine creates a new TimeStart engine. This engine can be used for fields such as
// flowEndSysUpTime etc.
func NewTimeEndEngine(params *TimeEndEngineParams, mgr *FieldEngineManager) (*TimeEndEngine, error) {
	o := new(TimeEndEngine)
	o.mgr = mgr
	err := o.validateParams(params)
	if err != nil {
		// validate has already set the errors of the manager
		return nil, err
	}
	o.par = params
	return o, nil
}

// validateParams validates that the parameters are correct.
func (o *TimeEndEngine) validateParams(params *TimeEndEngineParams) (err error) {
	if params.Size != 4 && params.Size != 8 {
		o.mgr.counters.invalidSize++
		err = fmt.Errorf("Size for this engine must be 4 or 8. Invalid size %v.", params.Size)
	}

	if params.DurationMin > params.DurationMax {
		o.mgr.counters.maxSmallerThanMin++
		err = fmt.Errorf("Duration min %v is greater than Duration max %v.", params.DurationMin, params.DurationMax)
	}

	if params.Size == 4 && params.DurationMax > 0xFFFFFFFF {
		o.mgr.counters.sizeTooSmall++
		err = fmt.Errorf("Size too small, can't represent max value %v  with %v bytes.", params.DurationMax, params.Size)
	}

	return err
}

// setCurrentFlowStartTime is a setter provided to the Time start engine to set the current flow timestamp.
func (o *TimeEndEngine) setCurrentFlowStartTime(currentFlowStartTime uint64) {
	o.currentFlowStart = currentFlowStartTime
}

// getFlowEndValue generates a relative timestamp for end of this timeflow.
func (o *TimeEndEngine) getFlowEndValue() (value uint64) {

	// Generates a uint64 with uniform distribution.
	// Converts the generated value to a value in the domain [min-max]. This way we get an ipg.
	genValue := rand.Uint64()
	duration := o.par.DurationMin + (genValue % (o.par.DurationMax - o.par.DurationMin + 1))
	// Assumes the TimeEnd engine has updated the previous flow end.
	value = o.currentFlowStart + duration

	return value
}

// Update implements the Update function of FieldEngineIF.
func (o *TimeEndEngine) Update(b []byte) (int, error) {

	// Get the Time Start engine. This can't be done upon creation as the engine might not be created yet.
	if o.timeStartEngine == nil {
		timeEndEngine, ok := o.mgr.engines[o.par.TimeStartEngineName]
		if !ok {
			return 0, fmt.Errorf("TimeStart engine name %v not found in engine manager database. Must provide this engine.", o.par.TimeStartEngineName)
		}
		o.timeStartEngine, ok = timeEndEngine.(*TimeStartEngine)
		if !ok {
			o.mgr.counters.badEngineType++
			return 0, fmt.Errorf("Failed converting engine %v to TimeStart. Make sure the engine type is corrent.", o.par.TimeStartEngineName)
		}
	}

	// Get a value for the end of this flow.
	value := o.getFlowEndValue()

	// Update the buffer with this value.
	switch o.par.Size {
	case 4:
		binary.BigEndian.PutUint32(b, uint32(value))
	case 8:
		binary.BigEndian.PutUint64(b, value)
	default:
		o.mgr.counters.invalidSize++
		return 0, errors.New("Size should be 4 or 8.")
	}

	// Update the start engine so it will know how to calculate the IPG.
	o.timeStartEngine.setPreviousFlowEndTime(value)
	return int(o.par.Size), nil
}

// GetOffset implements the GetOffset function of FieldEngineIF.
func (o *TimeEndEngine) GetOffset() uint16 {
	return o.par.Offset
}

// GetSize implements the GetSize function of FieldEngineIF.
func (o *TimeEndEngine) GetSize() uint16 {
	return o.par.Size
}

/* ------------------------------------------------------------------------------
							URL Histogram
--------------------------------------------------------------------------------*/
// HistogramURLEntry represents an URL which can be used as an entry for the
// HistogramEngine. This entry can be picked with probability prob.
type HistogramURLEntry struct {
	Schemes     []string `json:"schemes"`        // List of schemes
	Hosts       []string `json:"hosts"`          // List of hosts
	Paths       []string `json:"paths"`          // List of paths
	Queries     []string `json:"queries"`        // List of queries
	RandomQuery bool     `json:"random_queries"` // Should randomly generate queries
	Prob        uint32   `json:"prob"`           // Probability for this entry to be picked
}

// longestString receives a string slice and returns the longest string in that slice.
func longestString(stringList []string) (longest string) {
	for _, str := range stringList {
		if len(str) > len(longest) {
			longest = str
		}
	}
	return longest
}

// validateParams ensures a HistogramURLEntry is valid. For a valid entry we must have at
// least one scheme and one host. We need to validate that the longest url we can generate
// is not longer than the provided size.
func (o *HistogramURLEntry) validateParams(mgr *FieldEngineManager, size uint16) error {
	if o.Prob == 0 {
		// Probability 0 can be okay too but let's be strict and eliminate redundant values.
		mgr.counters.invalidHistogramEntry++
		return fmt.Errorf("Invalid probability 0, please remove this entry.")
	}
	if len(o.Schemes) == 0 {
		mgr.counters.emptyList++
		mgr.counters.invalidHistogramEntry++
		return fmt.Errorf("Schemes list can't be empty.")
	}
	if len(o.Hosts) == 0 {
		mgr.counters.emptyList++
		mgr.counters.invalidHistogramEntry++
		return fmt.Errorf("Hosts list can't be empty.")
	}
	if len(o.Queries) != 0 && o.RandomQuery {
		mgr.counters.invalidHistogramEntry++
		return fmt.Errorf("Can't have a list of queries and random queries set together. Please use only one.")
	}
	longestScheme := longestString(o.Schemes)
	longestHost := longestString(o.Hosts)
	longestPath := longestString(o.Paths)
	if !o.RandomQuery {
		longestQuery := longestString(o.Queries)
		u := &url.URL{
			Scheme:   longestScheme,
			Host:     longestHost,
			Path:     longestPath,
			RawQuery: longestQuery,
		}
		urlString := u.String()
		byteString := []byte(urlString)
		if len(byteString) > int(size) {
			mgr.counters.sizeTooSmall++
			return fmt.Errorf("Size %v is not enough to encode the longest URL %v. Please provide size of at least %v bytes.", size, urlString, len(byteString))
		}
	} else {
		u := &url.URL{
			Scheme: longestScheme,
			Host:   longestHost,
			Path:   longestPath,
		}
		urlString := u.String()
		byteString := []byte(urlString)
		if len(byteString)+3 > int(size) {
			// + 3 for `?q=`` and an empty query
			mgr.counters.sizeTooSmall++
			return fmt.Errorf("Size %v is not enough to encode the longest URL %v with a random query. Please provide size of at least %v bytes.", size, urlString, len(byteString)+3)
		}
	}
	return nil
}

func (o *HistogramURLEntry) getQuery(size uint16) (query string) {
	if len(o.Queries) != 0 {
		// queries doesn't necessarily need to be provided.
		if rand.Intn(len(o.Queries)+1) != len(o.Queries) {
			// Even if queries are provided, we don't need to choose them (empty query can be fine too).
			// Pick the empty query with uniform distribution over other queries.
			query = o.Queries[rand.Intn(len(o.Queries))]
		}
	} else if o.RandomQuery == true {
		// build random query of random size
		querySize := rand.Intn(int(size + 1)) // + 1 to generate in [0, n] instead of [0, n)
		dictionary := "aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ0123456789"
		var s strings.Builder
		s.Grow(querySize)
		for i := 0; i < querySize; i++ {
			rndChar := dictionary[rand.Intn(len(dictionary))]
			fmt.Fprintf(&s, "%c", rndChar)
		}
		query = s.String()
	}
	return query
}

// generateURL generates randomly an URL from the list of given schemes, hosts, paths and queries.
// Paths and queries can be picked or not, while schemes and hosts and always picked to be part of the url.
// All the generations are done uniformly from the lists.
func (o *HistogramURLEntry) generateURL(size uint16) *url.URL {
	var scheme, host, path string
	scheme = o.Schemes[rand.Intn(len(o.Schemes))]
	host = o.Hosts[rand.Intn(len(o.Hosts))]
	if len(o.Paths) != 0 {
		// paths doesn't necessarily need to be provided.
		if rand.Intn(len(o.Paths)+1) != len(o.Paths) {
			// Even if paths are provided, we don't need to choose them (empty path can be fine too).
			// Pick the empty path with uniform distribution over other paths.
			path = o.Paths[rand.Intn(len(o.Paths))]
		}
	}
	u := &url.URL{
		Scheme: scheme,
		Host:   host,
		Path:   path,
	}
	q := u.Query()
	queryMaxSize := size - uint16(len([]byte(u.String()))) - 3 // -2 for encoding ?q= in query
	query := o.getQuery(queryMaxSize)
	if o.RandomQuery == true {
		q.Set("q", query)
		u.RawQuery = q.Encode()
	} else {
		u.RawQuery = query
	}

	return u

}

// GetValue gets a newly generated URL randomly and returns it as a byte buffer.
func (o *HistogramURLEntry) GetValue(size uint16) (b []byte, err error) {
	url := o.generateURL(size)
	urlBytes := []byte(url.String())

	if int(size) < len(urlBytes) {
		return nil, fmt.Errorf("Size %v is too small for generated url %v.", size, url.String())
	}

	return urlBytes, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramURLEntry) GetProb() uint32 {
	return o.Prob
}

// HistogramURLParams is used for parsing the input JSON and create the entries.
type HistogramURLParams struct {
	HistogramEngineCommonParams                     // common params
	Entries                     []HistogramURLEntry `json:"entries"` // slice of entries
}

// CreateHistogramURLEngine creates an Histogram Engine whose entries are URL builders.
func CreateHistogramURLEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {
	// unmarshal the data
	p := HistogramURLParams{}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		// validate entries.
		err := p.Entries[i].validateParams(mgr, p.Size)
		if err != nil {
			// errors are already set in the entry.
			return nil, err
		}
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}

	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

/* ------------------------------------------------------------------------------
							String Histogram
--------------------------------------------------------------------------------*/
// HistogramStringEntry represents a string which can be used as an entry for the
// HistogramEngine can be picked with probability prob.
type HistogramStringEntry struct {
	Str          string `json:"str"`           // String value
	Prob         uint32 `json:"prob"`          // Probability for this entry to be picked
	PaddingValue uint8  `json:"padding_value"` // Value which will be used to pad the string to a fixed size. (In case it is needed)
	processedStr []byte // Process string into a byte for fast update. Pay the price once in processing.
}

// validateParams validates that a HistogramStringEntry is valid. An entry is considered valid
// if it can be successfully encoded to a byte slice within the given size.
func (o *HistogramStringEntry) validateParams(mgr *FieldEngineManager, size uint16) error {
	if o.Prob == 0 {
		// Probability 0 can be okay too but let's be strict and eliminate redundant values.
		mgr.counters.invalidHistogramEntry++
		return fmt.Errorf("Invalid probability 0, please remove this entry.")
	}
	byteString := []byte(o.Str)
	if len(byteString) > int(size) {
		mgr.counters.sizeTooSmall++
		return fmt.Errorf("String %v cannot be represented with size %v.", o.Str, size)
	}
	return nil
}

// processString processes the string ahead (upon creation) and creates the processedStr byte slice. This byte slice
// contains the value that this entry returns if picked. The byte slice can be a simple convertion from the string
// or a padded one in case padding is needed.
func (o *HistogramStringEntry) processString(shouldPad bool, size uint16) {
	o.processedStr = []byte(o.Str)
	if shouldPad {
		appendSize := int(size) - len(o.processedStr)
		for j := 0; j < appendSize; j++ {
			o.processedStr = append(o.processedStr, byte(o.PaddingValue))
		}
	}
}

// GetValue returns the picked string encoded into the byte parameter.
func (o *HistogramStringEntry) GetValue(size uint16) (b []byte, err error) {
	if int(size) < len(o.processedStr) {
		return nil, fmt.Errorf("Size %v is too small for string %v.", size, o.Str)
	}
	return o.processedStr, nil
}

// GetProb returns the probability for this entry to be picked in the histogram engine.
func (o *HistogramStringEntry) GetProb() uint32 {
	return o.Prob
}

// HistogramStringParams is used for parsing the input JSON and create the entries.
type HistogramStringParams struct {
	HistogramEngineCommonParams                        // common params
	ShouldPad                   bool                   `json:"should_pad"` // should pad the strings to given size. Defaults to false.
	Entries                     []HistogramStringEntry `json:"entries"`    // slice of entries
}

// CreateHistogramStringEngine creates an Histogram Engine whose entries are strings with a probability to get chosen.
func CreateHistogramStringEngine(params *fastjson.RawMessage, mgr *FieldEngineManager) (eng FieldEngineIF, err error) {

	// unmarshal the data
	p := HistogramStringParams{ShouldPad: false}
	err = fastjson.Unmarshal(*params, &p)
	if err != nil {
		mgr.counters.invalidJson++
		return nil, err
	}

	// create general params
	var histParams HistogramEngineParams
	histParams.Size = p.Size
	histParams.Offset = p.Offset
	for i := range p.Entries {
		// validate entries.
		err := p.Entries[i].validateParams(mgr, p.Size)
		if err != nil {
			// errors are already set in the entry.
			return nil, err
		}
		p.Entries[i].processString(p.ShouldPad, p.Size) // Create a padded string if necessary.
		histParams.Entries = append(histParams.Entries, &p.Entries[i])
	}
	// create and return new engine
	return NewHistogramEngine(&histParams, mgr)
}

func init() {
	// Register all the field engine types.
	fieldEngineRegister("uint", CreateUIntEngine)
	fieldEngineRegister("int", CreateIntEngine)
	fieldEngineRegister("float", CreateFloatEngine)
	fieldEngineRegister("uint_list", CreateUIntListEngine)
	fieldEngineRegister("int_list", CreateIntListEngine)
	fieldEngineRegister("float_list", CreateFloatListEngine)
	fieldEngineRegister("string_list", CreateStringListEngine)
	fieldEngineRegister("histogram_uint", CreateHistogramUInt32Engine)
	fieldEngineRegister("histogram_int", CreateHistogramInt32Engine)
	fieldEngineRegister("histogram_uint_range", CreateHistogramUInt32RangeEngine)
	fieldEngineRegister("histogram_int_range", CreateHistogramInt32RangeEngine)
	fieldEngineRegister("histogram_uint_list", CreateHistogramUInt32ListEngine)
	fieldEngineRegister("histogram_int_list", CreateHistogramInt32ListEngine)
	/*
		uint64 is an exception and needs special handlers
	*/
	fieldEngineRegister("histogram_uint64", CreateHistogramUInt64Engine)
	fieldEngineRegister("histogram_int64", CreateHistogramInt64Engine)
	fieldEngineRegister("histogram_uint64_range", CreateHistogramUInt64RangeEngine)
	fieldEngineRegister("histogram_int64_range", CreateHistogramInt64RangeEngine)
	fieldEngineRegister("histogram_uint64_list", CreateHistogramUInt64ListEngine)
	fieldEngineRegister("histogram_int64_list", CreateHistogramInt64ListEngine)

	fieldEngineRegister("time_start", CreateTimeStartEngine)
	fieldEngineRegister("time_end", CreateTimeEndEngine)

	fieldEngineRegister("histogram_url", CreateHistogramURLEngine)
	fieldEngineRegister("histogram_string", CreateHistogramStringEngine)
}
