How to handle C++ templates

[darmie]

I have a C++ code

#ifdef __cplusplus
extern "C" {
#endif

#ifdef _WIN32
  #define LIB_EXPORT __declspec(dllexport)
#else
  #define LIB_EXPORT
#endif

#include <stdlib.h>

#ifdef HXCPP
#include <hxcpp.h>
#endif


class leb128 {
    public: 
    template<typename int_t = uint64_t>
    LIB_EXPORT size_t encodeVarint(int_t value, Array<unsigned char> output) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
    }

    template<typename int_t = uint64_t>
    LIB_EXPORT int_t decodeVarint(unsigned char *input, size_t inputSize) {
        int_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
    }
};

#ifdef __cplusplus
}
#endif

I can call decodeVarint like this decodeVarint<uint32_t>(input, size);

How do I do this with Haxe & ammer ?

[Aurel300]

First, a small point – encodeVarint should not use Array, or any other hxcpp-specific feature.

We need to get rid of the extern "C" { wrapper, since templates are not supported in C linkage. Changing the interface to:

class leb128 {
    public: 
    template<typename int_t = uint64_t>
    size_t encodeVarint(int_t value, unsigned char *output);

    template<typename int_t = uint64_t>
    int_t decodeVarint(unsigned char *input, size_t inputSize);
};

Compiling the above will not produce any usable symbols, since the compiler has no way of knowing which specialisations it should instantiate, and the C linkage does not support templates. We can be explicit about which symbols we want to export:

template LIB_EXPORT size_t leb128::encodeVarint(uint64_t value, unsigned char *output);
template LIB_EXPORT size_t leb128::encodeVarint(uint32_t value, unsigned char *output);
// etc

Compiling now gives a library with some symbols:

$ g++ -std=c++11 -dynamiclib -o libtemplates.dylib templates.cpp
$ nm -gU libtemplates.dylib 
0000000000000f10 T __ZN6leb12812encodeVarintIjEEmT_Ph
0000000000000e70 T __ZN6leb12812encodeVarintIyEEmT_Ph

Since we got rid of the extern "C", the function names are mangled:

$ c++filt __ZN6leb12812encodeVarintIjEEmT_Ph __ZN6leb12812encodeVarintIyEEmT_Ph
unsigned long leb128::encodeVarint<unsigned int>(unsigned int, unsigned char*)
unsigned long leb128::encodeVarint<unsigned long long>(unsigned long long, unsigned char*)

---

There are some things ammer will need to do to support this:

• syntax to specify C++ template functions, e.g.

  public static function encodeVarint<T:ammer.ffi.CppTemplate>(value:T, output:Bytes):Int;

  • possibly some way to constraint the type (ammer.ffi.CppTemplate<[Int, Int64]>?)
  • alternatively enumerate the various specialisations, but function naming would be a bit annoying
• allow C++ linkage in general
  • might need to change the compilation process a bit, since currently HL and Eval stubs create C files
  • maybe simply mangle function names with the correct mangling rules; maybe C code could call them correctly then

Tl;dr: it's not possible yet, but it seems useful and not too difficult to implement.

[darmie]

Okay thanks

[darmie]

So I made all the changes necessary.

I am getting linker error while building on Mac.
All I have is a single header:

#ifdef __cplusplus
extern "C" {
#endif

#ifdef _WIN32
  #define LIB_EXPORT __declspec(dllexport)
#else
  #define LIB_EXPORT
#endif

#include <stdlib.h>


LIB_EXPORT uint32_t decodeVarUint32(unsigned char *input, size_t inputSize) {
        uint32_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
}
LIB_EXPORT int32_t decodeVarInt32(unsigned char *input, size_t inputSize) {
        int32_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
}

LIB_EXPORT uint64_t decodeVarUint64(unsigned char *input, size_t inputSize) {
        uint64_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
}
LIB_EXPORT int64_t decodeVarInt64(unsigned char *input, size_t inputSize) {
        int64_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
}

LIB_EXPORT uint8_t decodeVarUint8(unsigned char *input, size_t inputSize) {
        uint8_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
}
LIB_EXPORT int8_t decodeVarInt8(unsigned char *input, size_t inputSize) {
        int8_t ret = 0;
        for (size_t i = 0; i < inputSize; i++) {
            ret |= (input[i] & 127) << (7 * i);
            //If the next-byte flag is set
            if(!(input[i] & 128)) {
                break;
            }
        }
        return ret;
}



LIB_EXPORT size_t encodeVarInt32(int32_t value, unsigned char *output, size_t inputSize) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
}

LIB_EXPORT size_t encodeVarUint32(uint32_t value, unsigned char *output, size_t inputSize) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
}

LIB_EXPORT size_t encodeVarUint64(uint64_t value, unsigned char *output, size_t inputSize) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
}

LIB_EXPORT size_t encodeVarInt64(int64_t value, unsigned char *output, size_t inputSize) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
}

LIB_EXPORT size_t encodeVarUint8(uint8_t value, unsigned char *output, size_t inputSize) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
}

LIB_EXPORT size_t encodeVarInt8(int8_t value, unsigned char *output, size_t inputSize) {
        size_t outputSize = 0;
        //While more than 7 bits of data are left, occupy the last output byte
        // and set the next byte flag
        while (value > 127) {
            //|128: Set the next byte flag
            output[outputSize] = ((uint8_t)(value & 127)) | 128;
            //Remove the seven bits we just wrote
            value >>= 7;
            outputSize++;
        }
        output[outputSize++] = ((uint8_t)value) & 127;
        return outputSize;
}

#ifdef __cplusplus
}
#endif

Then implemented in Haxe like this:

private class Leb128Native extends Library<"leb128"> {
    public static inline function decodeVarUint32(input:Bytes, size:SizeOf<"input">):Int;
    public static inline function decodeVarUint64(input:Bytes, size:SizeOf<"input">):Int;

    public static inline function decodeVarInt32(input:Bytes, size:SizeOf<"input">):Int;
    public static inline function decodeVarInt64(input:Bytes, size:SizeOf<"input">):Int;


    public static inline function encodeVarUint32(value:Int, output:Bytes, size:SizeOf<"output">):Int;
    public static inline function encodeVarUint64(value:Int, output:Bytes, size:SizeOf<"output">):Int;

    public static inline function encodeVarInt32(value:Int, output:Bytes, size:SizeOf<"output">):Int;
    public static inline function encodeVarInt64(value:Int, output:Bytes, size:SizeOf<"output">):Int;

}

Error output:

Link: Test
ld: warning: ignoring file ../native/cpp/libleb128.dylib, file was built for unsupported file format ( 0x43 0x50 0x43 0x48 0x01 0x08 0x00 0x00 0xCE 0x0A 0x00 0x00 0x07 0xC1 0xB3 0xD0 ) which is not the architecture being linked (x86_64): ..//native/cpp/libleb128.dylib
duplicate symbol _encodeVarUint8 in:
    obj/darwin64/0f35468b_Leb128.o
    obj/darwin64/1f90944d_Leb128Native.o
duplicate symbol _decodeVarUint8 in:
    obj/darwin64/0f35468b_Leb128.o
    obj/darwin64/1f90944d_Leb128Native.o
...
...
ld: 36 duplicate symbols for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
Error: Build failed