xxHash - 编译, 库用法

xxHash - 编译, 库用法

概述

给AES设置 key, iv时, 如果长度不够, 需要填充.
刚开始实现填充时, 用的固定内容, 感觉不太好.
看到github上有个xxHash工程, 很多星, 准备用这个工程来生成固定长度的hash来填充key, iv. 这样填充内容的随机性和确定性就好很多.
不管原始key, iv的长度多少, 都先用xxHash处理成符合AES要求的key, iv长度的buf, 再调用AES加解密函数, 这样就不用考虑key, iv的填充问题了.

笔记

xxHash的代码库地址

https://github.com/Cyan4973/xxHash.git

编译

迁出到本地
打开普通的cmd命令行

cd /d D:\3rd_prj\crypt\xxHash\cmake_unofficial
mkdir .\build
cd .\build
cmake -G "Visual Studio 16 2019" -A x64 ..
cmake --build .
cmake --build . --target install

库安装后的路径为 C:\Program Files\xxHash
目录内容如下:

C:\Program Files\xxHash>tree /F
文件夹 PATH 列表
卷序列号为 BA70-59B2
C:.
├─bin
│      xxhash.dll
│      xxhsum.exe
│
├─include
│      xxh3.h
│      xxhash.h
│
├─lib
│  │  xxhash.lib
│  │
│  ├─cmake
│  │  └─xxHash
│  │          xxHashConfig.cmake
│  │          xxHashConfigVersion.cmake
│  │          xxHashTargets-debug.cmake
│  │          xxHashTargets.cmake
│  │
│  └─pkgconfig
│          libxxhash.pc
│
└─share
    └─man
        └─man1
                xxhsum.1

现在就可以拿 /lib/xxhash.lib, /include/*.h, /bin/xxhash.dll包含到自己工程干活了.

xxHash的用法

看 xxhsum.exe的实现.
在编译目录中, 有xxhsum的VS工程实现.

打开 xxHash.sln

将xxhsum设置活动工程.
找到程序入口

/*
 * The preferred method of obtaining the real UTF-16 arguments. Always works
 * on MSVC, sometimes works on MinGW-w64 depending on the compiler flags.
 */
#ifdef __cplusplus
extern "C"
#endif
int __cdecl wmain(int argc, wchar_t* utf16_argv[])
{
    return XSUM_wmain(argc, utf16_argv);
}
#else /* !XSUM_WIN32_USE_WMAIN */

下断点, 单步单步xxhsum的实现, 大致看一下.

自己搭建一个测试工程

用安装后的xxHash库和xxhsum工程实现, 自己搭建一个独立测试工程.
整了一个命令行工程(cosole x64 debug), 将库拷贝到自己工程, 设置头文件包含路径和库路径
先加入xxhsum.c, 尝试编译, 确啥补啥.
官方cli工程里面包含xxhash.h时, 都是用…/xxhash.h, 改为xxhash.h
编译通过, 功能正常.

测试工程文件目录

D:\my_dev\my_local_git_prj\study\xxHash\my_xxhsum>tree /F
文件夹 PATH 列表
卷序列号为 36AD-51CE
D:.
│  my_xxhsum.sln
│  my_xxhsum.vcxproj
│  my_xxhsum.vcxproj.filters
│  my_xxhsum.vcxproj.user
│  xsum_arch.h
│  xsum_bench.c
│  xsum_bench.h
│  xsum_config.h
│  xsum_os_specific.c
│  xsum_os_specific.h
│  xsum_output.c
│  xsum_output.h
│  xsum_sanity_check.c
│  xsum_sanity_check.h
│  xxhsum.c
│
└─xxHash_lib
    ├─bin
    │      xxhash.dll
    │      xxhsum.exe
    │
    ├─include
    │      xxh3.h
    │      xxhash.h
    │
    ├─lib
    │  │  xxhash.lib
    │  │
    │  ├─cmake
    │  │  └─xxHash
    │  │          xxHashConfig.cmake
    │  │          xxHashConfigVersion.cmake
    │  │          xxHashTargets-debug.cmake
    │  │          xxHashTargets.cmake
    │  │
    │  └─pkgconfig
    │          libxxhash.pc
    │
    └─share
        └─man
            └─man1
                    xxhsum.1

程序参数怎么给?

static int XSUM_usage(const char* exename)
{
    XSUM_log( WELCOME_MESSAGE(exename) );
    XSUM_log( "Print or verify checksums using fast non-cryptographic algorithm xxHash \n\n" );
    XSUM_log( "Usage: %s [options] [files] \n\n", exename);
    XSUM_log( "When no filename provided or when '-' is provided, uses stdin as input. \n");
    XSUM_log( "\nOptions: \n");
    XSUM_log( "  -H#          select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo);
    XSUM_log( "               0: XXH32 \n");
    XSUM_log( "               1: XXH64 \n");
    XSUM_log( "               2: XXH128 (also called XXH3_128bits) \n");
    XSUM_log( "               3: XXH3 (also called XXH3_64bits) \n");
    XSUM_log( "  -c, --check  read xxHash checksum from [files] and check them \n");
    XSUM_log( "  -h, --help   display a long help page about advanced options \n");
    return 0;
}


static int XSUM_usage_advanced(const char* exename)
{
    XSUM_usage(exename);
    XSUM_log( "\nAdvanced :\n");
    XSUM_log( "  -V, --version        Display version information \n");
    XSUM_log( "      --tag            Produce BSD-style checksum lines \n");
    XSUM_log( "      --little-endian  Checksum values use little endian convention (default: big endian) \n");
    XSUM_log( "      --binary         Read in binary mode \n");
    XSUM_log( "  -b                   Run benchmark \n");
    XSUM_log( "  -b#                  Bench only algorithm variant # \n");
    XSUM_log( "  -i#                  Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT);
    XSUM_log( "  -q, --quiet          Don't display version header in benchmark mode \n");
    XSUM_log( "\n");
    XSUM_log( "The following five options are useful only when verifying checksums (-c): \n");
    XSUM_log( "  -q, --quiet          Don't print OK for each successfully verified file \n");
    XSUM_log( "      --status         Don't output anything, status code shows success \n");
    XSUM_log( "      --strict         Exit non-zero for improperly formatted checksum lines \n");
    XSUM_log( "      --warn           Warn about improperly formatted checksum lines \n");
    XSUM_log( "      --ignore-missing Don't fail or report status for missing files \n");
    return 0;
}

给出不同命令行参数, 试试效果.

-H3

-H3 D:\my_tmp\my_xxhsum.pdb
结果如下:
\XXH3_82fc89454f8a2238  D:\\my_tmp\\my_xxhsum.pdb

-H3 --binary

-H3 --binary D:\my_tmp\my_xxhsum.pdb
结果如下:
\XXH3_82fc89454f8a2238  D:\\my_tmp\\my_xxhsum.pdb

看来都是2进制读取文件内容并计算hash.
单步时, 发现 --binary 是无效的, 被忽略掉了

 if (!strcmp(argument, "--binary")) { continue; } /* Just ignore it. See https://github.com/Cyan4973/xxHash/issues/812 */

算hash的cli函数

return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess, convention);

看到使用xxHash库的最终用法了

/*
 * XSUM_hashStream:
 * Reads data from `inFile`, generating an incremental hash of type hashType,
 * using `buffer` of size `blockSize` for temporary storage.
 */
static Multihash
XSUM_hashStream(FILE* inFile,
                AlgoSelected hashType,
                void* buffer, size_t blockSize)
{
    XXH32_state_t state32;
    XXH64_state_t state64;
    XXH3_state_t  state3;

    /* Init */
    (void)XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED);
    (void)XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED);
    (void)XXH3_128bits_reset(&state3);

    /* Load file & update hash */
    {   size_t readSize;
        while ((readSize = fread(buffer, 1, blockSize, inFile)) > 0) {
            switch(hashType)
            {
            case algo_xxh32:
                (void)XXH32_update(&state32, buffer, readSize);
                break;
            case algo_xxh64:
                (void)XXH64_update(&state64, buffer, readSize);
                break;
            case algo_xxh128:
                (void)XXH3_128bits_update(&state3, buffer, readSize);
                break;
            case algo_xxh3:
                (void)XXH3_64bits_update(&state3, buffer, readSize);
                break;
            default:
                assert(0);
            }
        }
        if (ferror(inFile)) {
            XSUM_log("Error: a failure occurred reading the input file.\n");
            exit(1);
    }   }

    {   Multihash finalHash = {0};
        switch(hashType)
        {
        case algo_xxh32:
            finalHash.hash32 = XXH32_digest(&state32);
            break;
        case algo_xxh64:
            finalHash.hash64 = XXH64_digest(&state64);
            break;
        case algo_xxh128:
            finalHash.hash128 = XXH3_128bits_digest(&state3);
            break;
        case algo_xxh3:
            finalHash.hash64 = XXH3_64bits_digest(&state3);
            break;
        default:
            assert(0);
        }
        return finalHash;
    }
}

保存Hash值

    /* display Hash value in selected format */
    switch(hashType)
    {
    case algo_xxh32:
        {   XXH32_canonical_t hcbe32;
            (void)XXH32_canonicalFromHash(&hcbe32, hashValue.hash32);
            f_displayLine(fileName, &hcbe32, hashType);
            break;
        }
    case algo_xxh64:
        {   XXH64_canonical_t hcbe64;
            (void)XXH64_canonicalFromHash(&hcbe64, hashValue.hash64);
            f_displayLine(fileName, &hcbe64, hashType);
            break;
        }
    case algo_xxh128:
        {   XXH128_canonical_t hcbe128;
            (void)XXH128_canonicalFromHash(&hcbe128, hashValue.hash128);
            f_displayLine(fileName, &hcbe128, hashType);
            break;
        }
    case algo_xxh3:
        {   XXH64_canonical_t hcbe64;
            (void)XXH64_canonicalFromHash(&hcbe64, hashValue.hash64);
            f_displayLine(fileName, &hcbe64, hashType);
            break;
        }
    default:
        assert(0);  /* not possible */
    }

显示hash值

static void XSUM_printLine_GNU(const char* filename,
                               const void* canonicalHash, const AlgoSelected hashType)
{
    XSUM_printLine_GNU_internal(filename, canonicalHash, hashType, XSUM_display_BigEndian);
}

static void XSUM_printLine_GNU_internal(const char* filename,
                               const void* canonicalHash, const AlgoSelected hashType,
                               XSUM_displayHash_f f_displayHash)
{
    assert(0 <= hashType && (size_t)hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
    {   const size_t hashLength = XSUM_algoLength[hashType];
        const int needsEscape = XSUM_filenameNeedsEscape(filename);
        if (needsEscape) {
            XSUM_output("%c", '\\');
        }
        XSUM_displayPrefix(hashType);
        f_displayHash(canonicalHash, hashLength);
        XSUM_output("  ");
        XSUM_printFilename(filename, needsEscape);
        XSUM_output("\n");
}   }

static void XSUM_display_BigEndian(const void* ptr, size_t length)
{
    const XSUM_U8* const p = (const XSUM_U8*)ptr;
    size_t idx;
    for (idx=0; idx<length; idx++)
        XSUM_output("%02x", p[idx]);
}

懂了

自己写一个命令行工程, 对一个buffer来做hash

官方工程是针对文件做hash, 我要的是对buffer做hash.
通过单步, 这个库的用法已经清楚了. 自己来整一遍, 只不过, 我要的是对buffer做hash.
整完了, 好使

在xxHash基础上, 封装了3个应用接口:

bool buffer_hash_XXH3_32bits(uint8_t* pBuf, size_t nLenBuf, uint32_t& hash_4Byte);
bool buffer_hash_XXH3_64bits(uint8_t* pBuf, size_t nLenBuf, uint64_t& hash_8Byte);
bool buffer_hash_XXH3_128bits(uint8_t* pBuf, size_t nLenBuf, uint64_t& hash_H8Byte, uint64_t& hash_K8Byte);

用这3个应用接口来做流的hash, 就方便多了.

测试工程(vs2019 vc++ console)

// @file test_xxHash_form_buffer.cpp

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>

#define XXH_STATIC_LINKING_ONLY   // 在包含xxhash.h之前, 必须定义这个宏
#include "xxhash.h"

#pragma comment(lib, "xxhash.lib")

bool buffer_hash_XXH3_32bits(uint8_t* pBuf, size_t nLenBuf, uint32_t& hash_4Byte);
bool buffer_hash_XXH3_64bits(uint8_t* pBuf, size_t nLenBuf, uint64_t& hash_8Byte);
bool buffer_hash_XXH3_128bits(uint8_t* pBuf, size_t nLenBuf, uint64_t& hash_H8Byte, uint64_t& hash_K8Byte);

void test_hash_128();
void test_hash_64();
void test_hash_32();

int main()
{
	printf("test xxHash from buffer\n");

	test_hash_128();
	test_hash_64();
	test_hash_32();

	return 0;
}

void test_hash_128()
{
	uint64_t hash_H8Byte = 0;
	uint64_t hash_L8Byte = 0;

	char szBuf[0x100];
	memset(szBuf, 0, sizeof(szBuf));
	strcpy(szBuf, "hello xxHash 128"); // add vs option _CRT_SECURE_NO_WARNINGS
	size_t nLenBuf = strlen(szBuf);

	if (buffer_hash_XXH3_128bits((uint8_t*)szBuf, nLenBuf, hash_H8Byte, hash_L8Byte))
	{
		printf("hash value =  %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X",
			(uint8_t)(((hash_H8Byte >> (8 * 7)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 6)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 5)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 4)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 3)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 2)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 1)) & 0xff)),
			(uint8_t)(((hash_H8Byte >> (8 * 0)) & 0xff)),

			(uint8_t)(((hash_L8Byte >> (8 * 7)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 6)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 5)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 4)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 3)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 2)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 1)) & 0xff)),
			(uint8_t)(((hash_L8Byte >> (8 * 0)) & 0xff))
		);

		printf("\n");
	}
	else {
		printf("error\n");
	}
}

void test_hash_64()
{
	uint64_t hash_8Byte = 0;

	char szBuf[0x100];
	memset(szBuf, 0, sizeof(szBuf));
	strcpy(szBuf, "hello xxHash 64"); // add vs option _CRT_SECURE_NO_WARNINGS
	size_t nLenBuf = strlen(szBuf);

	if (buffer_hash_XXH3_64bits((uint8_t*)szBuf, nLenBuf, hash_8Byte))
	{
		printf("hash value =  %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X %2.2X",
			(uint8_t)(((hash_8Byte >> (8 * 7)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 6)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 5)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 4)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 3)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 2)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 1)) & 0xff)),
			(uint8_t)(((hash_8Byte >> (8 * 0)) & 0xff))
		);

		printf("\n");
	}
	else {
		printf("error\n");
	}
}

void test_hash_32()
{
	uint32_t hash_4Byte = 0;

	char szBuf[0x100];
	memset(szBuf, 0, sizeof(szBuf));
	strcpy(szBuf, "hello xxHash 32"); // add vs option _CRT_SECURE_NO_WARNINGS
	size_t nLenBuf = strlen(szBuf);

	if (buffer_hash_XXH3_32bits((uint8_t*)szBuf, nLenBuf, hash_4Byte))
	{
		printf("hash value =  %2.2X %2.2X %2.2X %2.2X",
			(uint8_t)(((hash_4Byte >> (8 * 3)) & 0xff)),
			(uint8_t)(((hash_4Byte >> (8 * 2)) & 0xff)),
			(uint8_t)(((hash_4Byte >> (8 * 1)) & 0xff)),
			(uint8_t)(((hash_4Byte >> (8 * 0)) & 0xff))
		);

		printf("\n");
	}
	else {
		printf("error\n");
	}
}

bool buffer_hash_XXH3_128bits(uint8_t* pBuf, size_t nLenBuf, uint64_t& hash_H8Byte, uint64_t& hash_K8Byte)
{
	bool b_rc = false;
	XXH3_state_t  state3;
	XXH128_hash_t finalHash;

	do {
		// size_t 没有负数
		if ((NULL == pBuf) || (nLenBuf <= 0))
		{
			break;
		}

		(void)XXH3_128bits_reset(&state3);
		(void)XXH3_128bits_update(&state3, pBuf, nLenBuf);
		finalHash = XXH3_128bits_digest(&state3);
		
		hash_H8Byte = finalHash.high64;
		hash_K8Byte = finalHash.low64;

		b_rc = true;
	} while (false);

	return b_rc;
}

bool buffer_hash_XXH3_64bits(uint8_t* pBuf, size_t nLenBuf, uint64_t& hash_8Byte)
{
	bool b_rc = false;
	XXH3_state_t  state3;
	XXH64_hash_t finalHash;

	do {
		// size_t 没有负数
		if ((NULL == pBuf) || (nLenBuf <= 0))
		{
			break;
		}

		(void)XXH3_128bits_reset(&state3);
		(void)XXH3_64bits_update(&state3, pBuf, nLenBuf);
		finalHash = XXH3_64bits_digest(&state3);

		hash_8Byte = finalHash;

		b_rc = true;
	} while (false);

	return b_rc;
}

bool buffer_hash_XXH3_32bits(uint8_t* pBuf, size_t nLenBuf, uint32_t& hash_4Byte)
{
	bool b_rc = false;
	XXH32_state_t state32;
	XXH32_hash_t finalHash;

	do {
		// size_t 没有负数
		if ((NULL == pBuf) || (nLenBuf <= 0))
		{
			break;
		}

		// #define XXHSUM32_DEFAULT_SEED 0                   /* Default seed for algo_xxh32 */
		(void)XXH32_reset(&state32, 0 /*XXHSUM32_DEFAULT_SEED*/);
		(void)XXH32_update(&state32, pBuf, nLenBuf);
		finalHash = XXH32_digest(&state32);

		hash_4Byte = finalHash;

		b_rc = true;
	} while (false);

	return b_rc;
}

END