C Backend API

This is a reference to utils::ValueVerifyFatal. CVMAPILoadModel().

int CVMAPILoadModel(const char *graph_json, int graph_strlen, const char *param_bytes, int param_strlen, void **net, int device_type, int device_id)
int CVMAPIFreeModel(void *net)
int CVMAPIInference(void *net, char *input_data, int input_len, char *output_data)
int CVMAPIGetInputLength(void *net, unsigned long long *size)
int CVMAPIGetInputTypeSize(void *net, unsigned long long *size)
int CVMAPIGetOutputLength(void *net, unsigned long long *size)
int CVMAPIGetOutputTypeSize(void *net, unsigned long long *size)
int CVMArrayAlloc(const cvm_index_t *shape, int ndim, int dtype_code, int dtype_bits, int dtype_lanes, int device_type, int device_id, CVMArrayHandle *out)

Allocate a nd-array’s memory, including space of shape, of given spec.

Parameters

  • shape – The shape of the array, the data content will be copied to out

  • ndim – The number of dimension of the array.

  • dtype_code – The type code of the dtype

  • dtype_bits – The number of bits of dtype

  • dtype_lanes – The number of lanes in the dtype.

  • device_type – The device type of context

  • device_id – The device id of context.

  • out – The output handle.

Returns

0 when success, -1 when failure happens

int CVMArrayFree(CVMArrayHandle handle)

Free the CVM Array.

Parameters

handle – The array handle to be freed.

Returns

0 when success, -1 when failure happens

struct CVMModel

Public Functions

CVMModel(const string &graph, DLContext _ctx)
~CVMModel()
int LoadParams(const string &params_str)
int LoadParamsFromFile(string filepath)
int GetInputLength()
int GetOutputLength()
int64_t GetStorageSize()
int64_t GetOps()
int GetSizeOfOutputType()
int GetSizeOfInputType()
void Run(DLTensor *input, std::vector<DLTensor*> output)
DLTensor *PlanInput(void*, int)
std::vector<DLTensor*> PlanOutput()
void SaveTensor(std::vector<DLTensor*> outputs, char *data)
std::string GetVersion()
std::string GetPostprocessMethod()
bool SetPostprocessMethod(const string postprocess_method)
bool IsReady() const
class CvmRuntime : public ModuleNode

Tiny graph runtime.

This runtime can be acccesibly in various language via CVM runtime PackedFunc API.

Public Functions

virtual PackedFunc GetFunction(const std::string &name) final

Get member function to front-end.

Parameters

  • name – The name of the function.

  • sptr_to_self – The pointer to the module node.

Returns

The corresponding member function.

inline const char *type_key() const final
Returns

The type key of the executor.

void SetGraph(const std::string &graph_json)

Initialize the graph executor with graph and context.

Parameters

  • graph_json – The execution graph.

  • module – The module containing the compiled functions for the host processor.

  • ctxs – The context of the host and devices where graph nodes will be executed on.

void GetShape(int index, DLTensor *data)

Get the input index given the name of input.

Parameters

name – The name of the input.

Returns

The index of input.

void SetData(int index, DLTensor *data_in)

set named data input to the graph.

Parameters

  • index – The input index.

  • data_in – The input data.

void SetInput(int index, DLTensor *data_in)

set index-th input to the graph.

Parameters

  • index – The input index.

  • data_in – The input data.

int NumOutputs() const

Get the number of outputs.

Returns

The number of outputs from graph.

NDArray GetInput(int index) const

Return NDArray for given input index.

Parameters

index – The input index.

Returns

NDArray corresponding to given input node index.

NDArray GetOutput(int index) const

Return NDArray for given output index.

Parameters

index – The output index.

Returns

NDArray corresponding to given output node index.

void CopyOutputTo(int index, DLTensor *data_out)

Copy index-th output to data_out.

Parameters

  • index – The output index.

  • data_out – the output data.

void LoadParams(utils::Stream *strm)

Load parameters from binary stream.

Parameters

strm – The input stream.

void LoadParams(const std::string &param_blob)

Load parameters from parameter blob.

Parameters

param_blob – A binary blob of parameter.

inline uint32_t GetNumOfNodes() const

Get total number of nodes.

Returns

Total number of nodes.

void Init()

Setup the shape, type, and precision.

void SetupStorage()

Setup the temporal storage.

void SetupOpExecs()

Setup the executors.

std::function<void()> CreateCVMOp(const CVMOpParam &param, NodeAttrs *attr, const std::vector<DLTensor> &args)

Create an execution function given input.

Parameters

  • attrs – The node attributes.

  • args – The arguments to the functor, including inputs and outputs.

  • num_inputs – Number of inputs.

Returns

The created executor.

Public Members

std::vector<Node> nodes_

The graph nodes.

std::vector<uint32_t> input_nodes_

The argument nodes.

std::vector<uint32_t> node_row_ptr_

Used for quick entry indexing.

std::vector<NodeEntry> outputs_

Output entries.

GraphAttr attrs_

Additional graph attributes.

cvm::runtime::Module module_

The code module that contains both host and device code.

std::vector<CVMContext> ctxs_

Execution context of all devices including the host.

std::vector<NDArray> storage_pool_

Common storage pool for all devices.

std::vector<NDArray> data_entry_

Data entry of each node.

int64_t extra_space_size_

Operator on each node.

void cvm::runtime::CvmRuntime::Setup()
void cvm::runtime::CvmRuntime::Run()
class Op

Operator structure.

Besides the fields in the structure, arbitary additional information can be associated with each op. See function GetAttr for details.

 // Example usage of Op

 // registeration of oeprators
 // NOTE that the attr function can register any
 // additional attributes to the operator
 CVM_REGISTER_OP(add)
 .describe("add two inputs together")
 .set_num_inputs(2)
 .set_attr<OpKernel>("OpKernel<gpu>", AddKernel)
 .include("ElementwiseOpAttr");

 // can register attribute by group
 // all the ops that include the group get the attribute.
 CVM_REGISTER_OP_GROUP(ElementwiseOpAttr)
 .set_attr<FInferShape>("FInferShape", ElementwiseInferShape);

 CVM_REGISTER_OP(sub)
 .describe("substract one tensor from another")
 .set_num_inputs(2);

 // Can call regster multiple times in different files
 // to register different part of information
 CVM_REGISTER_OP(sub)
 .set_attr<OpKernel>("OpKernel<gpu>", SubKernel);
 .include("ElementwiseOpAttr");

 // get operators from registry.
 void my_function() {
   const Op* add = Op::Get("add");
   const Op* sub = Op::Get("sub");
   // query basic information about each operator.
   assert(op->name == "plus");
   assert(op->num_inputs == 2);

   // get additional registered information,
   // Assume user registered a OpKernel type attribute as gpu_kernel on each operator.
   const OpMap<OpKernel>& kernel = Op::GetAttr<OpKernel>("OpKernel<gpu>");
   // we can get the kernel functions by using operator as key.
   auto add_kernel = kernel[add];
   auto sub_kernel = kernel[sub];
   // subsequent code can make use of the queried kernel functions.
}

Public Functions

inline Op &describe(const std::string &descr)

setter function during registration Set the description of operator

Parameters

descr – the description string.

Returns

reference to self.

inline Op &add_argument(const std::string &name, const std::string &type, const std::string &description)

Add argument information to the function.

Parameters

  • name – Name of the argument.

  • type – Type of the argument.

  • description – Description of the argument.

Returns

reference to self.

inline Op &add_arguments(const std::vector<ParamFieldInfo> &args)

Append list if arguments to the end.

Parameters

args – Additional list of arguments.

Returns

reference to self.

inline Op &set_num_inputs(uint32_t n)

Set the num_inputs.

Parameters

n – The number of inputs to be set.

Returns

reference to self.

inline Op &set_support_level(uint32_t level)

Set the support level of op.

Parameters

level – The support level.

Returns

reference to self.

inline Op &set_num_inputs(std::function<uint32_t(const NodeAttrs &attr)> fn)

Set the get_num_outputs function.

Parameters

fn – The function to be set.

Returns

reference to self.

inline Op &set_num_outputs(uint32_t n)

Set the num_outputs.

Parameters

n – The number of outputs to be set.

Returns

reference to self.

inline Op &set_num_outputs(std::function<uint32_t(const NodeAttrs &attr)> fn)

Set the get_num_outputs function.

Parameters

fn – The function to be set.

Returns

reference to self.

inline Op &set_attr_parser(std::function<void(NodeAttrs *attrs)> fn)

Set the attr_parser function.

Parameters

fn – The number of outputs to be set.

Returns

reference to self.

template<typename ValueType>
inline Op &set_attr(const std::string &attr_name, const ValueType &value, int plevel = 10)

Register additional attributes to operator.

Cannot set with same plevel twice in the code.

Parameters

  • attr_name – The name of the attribute.

  • value – The value to be set.

  • plevel – The priority level of this set, an higher priority level attribute will replace lower priority level attribute. Must be bigger than 0.

Template Parameters

ValueType – The type of the value to be set.

Op &add_alias(const std::string &alias)

Add another alias to this operator. The same Op can be queried with Op::Get(alias)

Parameters

alias – The alias of the operator.

Returns

reference to self.

Op &include(const std::string &group_name)

Include all the attributes from an registered op group.

See also

CVM_REGISTER_OP_GROUP

Parameters

group_name – The name of the group.

Returns

reference to self.

Public Members

std::string name

name of the operator

std::string description

detailed description of the operator This can be used to generate docstring automatically for the operator.

uint32_t num_inputs = 1

number of inputs to the operator, -1 means it is variable length When get_num_inputs is presented, the number will be decided by get_num_inputs instead.

See also

get_num_inputs

uint32_t num_outputs = 1

number of outputs of the operator When get_num_outputs is presented. The number of outputs will be decided by get_num_outputs function

See also

get_num_outputs

uint32_t support_level = 10

support level of the operator, The lower the more priority it contains. This is in analogies to BLAS levels.

std::function<uint32_t(const NodeAttrs &attrs)> get_num_outputs = nullptr

get number of outputs given information about the node.

Param attrs

The attribute of the node

Return

number of outputs.

std::function<uint32_t(const NodeAttrs &attrs)> get_num_inputs = nullptr

get number of inputs given information about the node.

Param attrs

The attribute of the node

Return

number of inputs

std::function<void(NodeAttrs *attrs)> attr_parser = nullptr

Attribute parser to parse the NodeAttrs information.

This can help to get quick access to a parsed attribute object

// Example usage of attr_parser.

// Suppose we want to register operator sum.
// The parameters about sum operator
struct SumParam {
  int axis;
};
// The parser function
void SumAttrParser(NodeAttrs* attrs) {
   // This will be invoked during node construction.
   SumParam param;
   // parse axis string to integer
   param.axis = atoi(attrs->dict["axis"].c_str());
   // set the parsed parameter
   attrs->parsed = std::move(param);
}
// The other function that can utilize the parsed result.
TShape SumInferShape(const NodeAttrs& attrs,
                     const std::vector<TShape>& ishapes) {
   // we can use the parsed version of param
   // without repeatively parsing the parameter
   const SumParam& param = cvm::get<SumParam>(attrs.parsed);
}

Public Static Functions

static const Op *Get(const std::string &op_name)

Get an Op for a given operator name. Will raise an error if the op has not been registered.

Parameters

op_name – Name of the operator.

Returns

Pointer to a Op, valid throughout program lifetime.

static const std::vector<std::string> GetAll()

Get all Ops.

Parameters

none. –

Returns

Vector of strings, include all Op names.

template<typename ValueType>
static inline const OpMap<ValueType> &GetAttr(const std::string &attr_name)

Get additional registered attribute about operators. If nothing has been registered, an empty OpMap will be returned.

Parameters

attr_name – The name of the attribute.

Template Parameters

ValueType – The type of the attribute.

Returns

An OpMap of specified attr_name.

CVM_REGISTER_OP(OpName)

Register a new operator, or set attribute of the corresponding op. 

CVM_REGISTER_OP(add)
.describe("add two inputs together")
.set_num_inputs(2)
.set_attr<OpKernel>("gpu_kernel", AddKernel);

Parameters

  • OpName – The name of registry

class LogMessageFatal
class ValueVerifyFatal
VERIFY(x)
VERIFY_LT(x, y)
VERIFY_GT(x, y)
VERIFY_LE(x, y)
VERIFY_GE(x, y)
VERIFY_EQ(x, y)
VERIFY_NE(x, y)
VERIFY_NOTNULL(x)
CHECK(x)
CHECK_LT(x, y)
CHECK_GT(x, y)
CHECK_LE(x, y)
CHECK_GE(x, y)
CHECK_EQ(x, y)
CHECK_NE(x, y)
CHECK_NOTNULL(x)