diff --git a/cachelab/Cache.c b/cachelab/Cache.c new file mode 100644 index 0000000..732bb47 --- /dev/null +++ b/cachelab/Cache.c @@ -0,0 +1,302 @@ +/////////////////////////////////////////////////////////////////////// +//// Copyright 2022 by mars. // +/////////////////////////////////////////////////////////////////////// + +#include +#include + +#include "common.h" + +#define DEBUG 0 + +#define GET_POWER_OF_2(X) (X == 0x00 ? 0 : \ + X == 0x01 ? 0 : \ + X == 0x02 ? 1 : \ + X == 0x04 ? 2 : \ + X == 0x08 ? 3 : \ + X == 0x10 ? 4 : \ + X == 0x20 ? 5 : \ + X == 0x40 ? 6 : \ + X == 0x80 ? 7 : \ + X == 0x100 ? 8 : \ + X == 0x200 ? 9 : \ + X == 0x400 ? 10 : \ + X == 0x800 ? 11 : \ + X == 0x1000 ? 12 : \ + X == 0x2000 ? 13 : \ + X == 0x4000 ? 14 : \ + X == 0x8000 ? 15 : \ + X == 0x10000 ? 16 : \ + X == 0x20000 ? 17 : \ + X == 0x40000 ? 18 : \ + X == 0x80000 ? 19 : \ + X == 0x100000 ? 20 : \ + X == 0x200000 ? 21 : \ + X == 0x400000 ? 22 : \ + X == 0x800000 ? 23 : \ + X == 0x1000000 ? 24 : \ + X == 0x2000000 ? 25 : \ + X == 0x4000000 ? 26 : \ + X == 0x8000000 ? 27 : \ + X == 0x10000000 ? 28 : \ + X == 0x20000000 ? 29 : \ + X == 0x40000000 ? 30 : \ + X == 0x80000000 ? 31 : \ + X == 0x100000000 ? 32 : 0) + +/* + 直接映射Data Cache，16KB大小 + 每行存放64个字节，共256行 +*/ +#define DCACHE_SIZE 16384 +#define DCACHE_DATA_PER_LINE 16 // 必须是8字节的倍数 +#define DCACHE_DATA_PER_LINE_ADDR_BITS GET_POWER_OF_2(DCACHE_DATA_PER_LINE) // 必须与上面设置一致，即64字节，需要6位地址 +#define DCACHE_SET (DCACHE_SIZE/DCACHE_DATA_PER_LINE) +#define DCACHE_SET_ADDR_BITS GET_POWER_OF_2(DCACHE_SET) // 必须与上面设置一致，即256行，需要8位地址 + +// Cache行的结构，包括Valid、Tag和Data。你所有的状态信息，只能记录在Cache行中！ +struct DCACHE_LineStruct +{ + UINT8 Valid; + UINT64 Tag; + UINT8 Data[DCACHE_DATA_PER_LINE]; +}DCache[DCACHE_SET]; + +/* + DCache初始化代码，一般需要把DCache的有效位Valid设置为0 + 模拟器启动时，会调用此InitDataCache函数 +*/ +void InitDataCache() +{ + UINT32 i; + printf("[%s] +-----------------------------------+\n", __func__); + printf("[%s] | 威震天的Data Cache初始化ing.... |\n", __func__); + printf("[%s] +-----------------------------------+\n", __func__); + for (i = 0; i < DCACHE_SET; i++) + DCache[i].Valid = 0; +} + +/* + 从Memory中读入一行数据到Data Cache中 +*/ +void LoadDataCacheLineFromMemory(UINT64 Address, UINT32 CacheLineAddress) +{ + // 一次性从Memory中将DCACHE_DATA_PER_LINE数据读入某个Data Cache行 + // 提供了一个函数，一次可以读入8个字节 + UINT32 i; + UINT64 ReadData; + UINT64 AlignAddress; + UINT64* pp; + + AlignAddress = Address & ~(DCACHE_DATA_PER_LINE - 1); // 地址必须对齐到DCACHE_DATA_PER_LINE (64)字节边界 + pp = (UINT64*)DCache[CacheLineAddress].Data; + for (i = 0; i < DCACHE_DATA_PER_LINE / 8; i++) + { + ReadData = ReadMemory(AlignAddress + 8LL * i); + if (DEBUG) + printf("[%s] Address=%016llX ReadData=%016llX\n", __func__, AlignAddress + 8LL * i, ReadData); + pp[i] = ReadData; + } + +} + +/* + 将Data Cache中的一行数据，写入存储器 +*/ +void StoreDataCacheLineToMemory(UINT64 Address, UINT32 CacheLineAddress) +{ + // 一次性将DCACHE_DATA_PER_LINE数据从某个Data Cache行写入Memory中 + // 提供了一个函数，一次可以写入8个字节 + UINT32 i; + UINT64 WriteData; + UINT64 AlignAddress; + UINT64* pp; + + AlignAddress = Address & ~(DCACHE_DATA_PER_LINE - 1); // 地址必须对齐到DCACHE_DATA_PER_LINE (64)字节边界 + pp = (UINT64*)DCache[CacheLineAddress].Data; + WriteData = 0; + for (i = 0; i < DCACHE_DATA_PER_LINE / 8; i++) + { + WriteData = pp[i]; + WriteMemory(AlignAddress + 8LL * i, WriteData); + if (DEBUG) + printf("[%s] Address=%016llX ReadData=%016llX\n", __func__, AlignAddress + 8LL * i, WriteData); + } +} + +/* + Data Cache访问接口，系统模拟器会调用此接口，来实现对你的Data Cache访问 + Address: 访存字节地址 + Operation: 操作：读操作（'L'）、写操作（'S'）、读-修改-写操作（'M'） + DataSize: 数据大小：1字节、2字节、4字节、8字节 + StoreValue: 当执行写操作的时候，需要写入的数据 + LoadResult: 当执行读操作的时候，从Cache读出的数据 +*/ +UINT8 AccessDataCache(UINT64 Address, UINT8 Operation, UINT8 DataSize, UINT64 StoreValue, UINT64* LoadResult) +{ + UINT32 CacheLineAddress; + UINT8 BlockOffset; + UINT64 AddressTag; + UINT8 MissFlag = 'M'; + UINT64 ReadValue; + + *LoadResult = 0; + + /* + * 直接映射中，Address被切分为 AddressTag，CacheLineAddress，BlockOffset + */ + + // CacheLineAddress Cache的行号，在直接映射中，就是组号（每组1行） + CacheLineAddress = (Address >> DCACHE_DATA_PER_LINE_ADDR_BITS) % DCACHE_SET; + BlockOffset = Address % DCACHE_DATA_PER_LINE; + AddressTag = (Address >> DCACHE_DATA_PER_LINE_ADDR_BITS) >> DCACHE_SET_ADDR_BITS; // 地址去掉DCACHE_SET、DCACHE_DATA_PER_LINE，剩下的作为Tag。警告！不能将整个地址作为Tag！！ + + if (DCache[CacheLineAddress].Valid == 1 && DCache[CacheLineAddress].Tag == AddressTag) + { + MissFlag = 'H'; // 命中！ + + if (Operation == 'L') // 读操作 + { + ReadValue = 0; + switch (DataSize) + { + case 1: // 1个字节 + ReadValue = DCache[CacheLineAddress].Data[BlockOffset + 0]; + break; + case 2: // 2个字节 + BlockOffset = BlockOffset & 0xFE; // 需对齐到2字节边界 + ReadValue = DCache[CacheLineAddress].Data[BlockOffset + 1]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 0]; + break; + case 4: // 4个字节 + BlockOffset = BlockOffset & 0xFC; // 需对齐到4字节边界 + ReadValue = DCache[CacheLineAddress].Data[BlockOffset + 3]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 2]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 1]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 0]; + break; + case 8: // 8个字节 + BlockOffset = BlockOffset & 0xF8; // 需对齐到8字节边界 + ReadValue = DCache[CacheLineAddress].Data[BlockOffset + 7]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 6]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 5]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 4]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 3]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 2]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 1]; ReadValue = ReadValue << 8; + ReadValue |= DCache[CacheLineAddress].Data[BlockOffset + 0]; + break; + } + *LoadResult = ReadValue; + if (DEBUG) + printf("[%s] Address=%016llX Operation=%c DataSize=%u StoreValue=%016llX ReadValue=%016llX\n", __func__, Address, Operation, DataSize, StoreValue, ReadValue); + } + else if (Operation == 'S' || Operation == 'M') // 写操作（修改操作在此等价于写操作） + { + if (DEBUG) + printf("[%s] Address=%016llX Operation=%c DataSize=%u StoreValue=%016llX\n", __func__, Address, Operation, DataSize, StoreValue); + switch (DataSize) + { + case 1: // 1个字节 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; + break; + case 2: // 2个字节 + BlockOffset = BlockOffset & 0xFE; // 需对齐到2字节边界 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 1] = StoreValue & 0xFF; + break; + case 4: // 4个字节 + BlockOffset = BlockOffset & 0xFC; // 需对齐到4字节边界 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 1] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 2] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 3] = StoreValue & 0xFF; + break; + case 8: // 8个字节 + BlockOffset = BlockOffset & 0xF8; // 需对齐到8字节边界 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 1] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 2] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 3] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 4] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 5] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 6] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 7] = StoreValue & 0xFF; + break; + } + } + } + else + { + if (DEBUG) + printf("[%s] Address=%016llX Operation=%c DataSize=%u StoreValue=%016llX\n", __func__, Address, Operation, DataSize, StoreValue); + MissFlag = 'M'; // 不命中 + if (DCache[CacheLineAddress].Valid == 1) + { + // 淘汰对应的Cache行，如果对应的Cache行有数据，需要写回到Memory中 + UINT64 OldAddress; + // OldAddress = > (Tag,Set,0000) + OldAddress = ((DCache[CacheLineAddress].Tag << DCACHE_SET_ADDR_BITS) << DCACHE_DATA_PER_LINE_ADDR_BITS) | ((UINT64)CacheLineAddress << DCACHE_DATA_PER_LINE_ADDR_BITS); // 从Tag中恢复旧的地址 + StoreDataCacheLineToMemory(OldAddress, CacheLineAddress); + } + // 需要从Memory中读入新的行（真实情况下，这个LoadCacheLineFromMemory需要很长时间的） + LoadDataCacheLineFromMemory(Address, CacheLineAddress); + DCache[CacheLineAddress].Valid = 1; + DCache[CacheLineAddress].Tag = AddressTag; + if (Operation == 'L') // 读操作 + { + // 读操作不需要做事情，因为已经MISS了 + } + else if (Operation == 'S' || Operation == 'M') // 写操作（修改操作在此等价于写操作） + { + // 写操作，需要将新的StoreValue更新到CacheLine中 + switch (DataSize) + { + case 1: // 1个字节 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; + break; + case 2: // 2个字节 + BlockOffset = BlockOffset & 0xFE; // 需对齐到2字节边界 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 1] = StoreValue & 0xFF; + break; + case 4: // 4个字节 + BlockOffset = BlockOffset & 0xFC; // 需对齐到4字节边界 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 1] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 2] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 3] = StoreValue & 0xFF; + break; + case 8: // 8个字节 + BlockOffset = BlockOffset & 0xF8; // 需对齐到8字节边界 + DCache[CacheLineAddress].Data[BlockOffset + 0] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 1] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 2] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 3] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 4] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 5] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 6] = StoreValue & 0xFF; StoreValue = StoreValue >> 8; + DCache[CacheLineAddress].Data[BlockOffset + 7] = StoreValue & 0xFF; + break; + } + } + } + return MissFlag; +} + +/* 指令Cache实现部分，可选实现 */ +void InitInstCache(void) +{ + return; +} + +void LoadInstCacheLineFromMemory(UINT64 Address, UINT32 CacheLineAddress) +{ + return; +} + +UINT8 AccessInstCache(UINT64 Address, UINT8 Operation, UINT8 InstSize, UINT64* InstResult) +{ + // 返回值'M' = Miss，'H'=Hit + return 'M'; +} \ No newline at end of file diff --git a/cachelab/CacheHelper.c b/cachelab/CacheHelper.c new file mode 100644 index 0000000..4aaa3b6 --- /dev/null +++ b/cachelab/CacheHelper.c @@ -0,0 +1,792 @@ +/////////////////////////////////////////////////////////////////////// +//// Copyright 2022 by mars. // +/////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include "getopt.h" +#include "cbsl.h" + +#define CBSL_ERROR_CHECK(X) {if ((X) == cbsl_error) { fprintf(stderr, "error: %s\n", (#X)); }} + +/* +fcn ideal% #items #buckets dup% fl add_usec find_usec del-all usec +--- ------ ---------- ---------- ----- -- ---------- ---------- ------------ +OAT 89.1% 597731 524288 0% ok 317292 179676 59726 +FNV 88.6% 597731 262144 0% ok 226650 220626 61619 +JEN 87.8% 597731 524288 0% ok 321989 175945 59956 +BER 86.4% 597731 262144 0% ok 198477 179332 60901 +SAX 70.4% 597731 524288 0% ok 270281 196427 64064 +SFH 69.2% 597731 524288 0% ok 289843 165105 61860 +FNV1A_Pippip_Yurii 更快？？ +*/ + +// 使用OAT hash算法，以获得最高性能 +#include "uthash.h" +#undef HASH_FUNCTION +//#define HASH_FUNCTION HASH_OAT +#define HASH_FUNCTION(keyptr,keylen,hashv) (hashv) = FNV1A_Pippip_Yurii((UINT64 *)keyptr) + +#define VERBOSE_MSG 0 + +#ifdef _WIN32 +#define ZLIB_WINAPI +#endif + +#include "common.h" + +#ifdef _WIN64 +#pragma comment(lib,"..\\zstd\\libzstd_static-win64.lib") +#elif _WIN32 +#pragma comment(lib,"..\\zstd\\libzstd_static-win32.lib") +#endif + +/* + 构建一个用Hash表访问的Memory，以便能够紧凑存放稀疏、随机的Memory数据 +*/ + +struct MemoryDataStruct +{ + UINT64 Address; // 地址（默认是64位字地址） + UINT64 Data; // 数据（默认保存64位字数据） + UT_hash_handle memory; // makes this structure hashable +} *MemoryHash = NULL; + +static void WriteMemoryHash(UINT64 Address, UINT64 WriteValue, UINT8 WriteSize); +static UINT64 ReadMemoryHash(UINT64 Address, UINT8 ReadSize); + +#define MEMORY_TRACE_CHUNK (1<<20) // MemoryTrace结构，每次增长1M条记录 +struct MemoryTraceStruct +{ + UINT8 Operation; + UINT64 Address; + UINT8 Size; +} *MemoryTrace; +UINT64 MemoryTraceCounter, MemoryTraceCapacity; + +/* +* 统计计数器 +*/ +UINT64 GlobalMemoryModifyCounter; +UINT64 GlobalMemoryInstCounter; +UINT64 GlobalMemoryWriteCounter; +UINT64 GlobalCacheWriteHitCounter; +UINT64 GlobalMemoryReadCounter; +UINT64 GlobalCacheReadHitCounter; +UINT64 GlobalCacheModifyHitCounter; +UINT64 GlobalSimReadMemoryCounter; +UINT64 GlobalSimWriteMemoryCounter; + +UINT64 GlobalCacheInstHitCounter; + +#define CHUNK 16384 + +static void Trim(char* src) +{ + char* begin = src; + char* end = src + strlen(src); + + if (begin == end) return; + + while (*begin == ' ' || *begin == '\t') + ++begin; + while ((*end) == '\0' || *end == ' ' || *end == '\t' || *end == '\n' || *end == '\r') + --end; + + if (begin > end) { + *src = '\0'; return; + } + while (begin != end) { + *src++ = *begin++; + } + + *src++ = *end; + *src = '\0'; + + return; +} + +typedef struct { + const char* start; + size_t len; +} token; + +// https://stackoverflow.com/a/39286524 +static char** split_space(const char* str, int* field_count) +{ + char** array; + unsigned int start = 0, stop, toks = 0, t; + token* tokens = malloc((strlen(str) + 1) * sizeof(token)); + for (stop = 0; str[stop]; stop++) { + if (str[stop] == ' ' || str[stop] == '\t' || str[stop] == ',') + { + tokens[toks].start = str + start; + tokens[toks].len = stop - start; + toks++; + start = stop + 1; + } + } + /* Mop up the last token */ + tokens[toks].start = str + start; + tokens[toks].len = stop - start; + toks++; + array = malloc(toks * sizeof(char*)); + for (t = 0; t < toks; t++) { + /* Calloc makes it nul-terminated */ + char* token = calloc(tokens[t].len + 1, 1); + memcpy(token, tokens[t].start, tokens[t].len); + array[t] = token; + } + free(tokens); + *field_count = toks; + return array; +} + +#define SR_A ((UINT64)1103515245) +#define SR_C ((UINT64)12345) +#define SR_M ((UINT64)1<<32) +UINT64 Xn; + +static void SyncRand(UINT64 Seed) +{ + Xn = Seed; +} + +static UINT8 GetRand8() +{ + Xn = ((SR_A * Xn + SR_C) % SR_M); + return (UINT8)Xn; +} + +static UINT16 GetRand16() +{ + Xn = ((SR_A * Xn + SR_C) % SR_M); + return (UINT16)Xn; +} + +static UINT32 GetRand32() +{ + Xn = ((SR_A * Xn + SR_C) % SR_M); + return (UINT32)Xn; +} + +static UINT64 GetRand64() +{ + UINT32 Hi, Lo; + Hi = GetRand32(); + Lo = GetRand32(); + + return ((UINT64)Hi << 32 | Lo); +} + +//https://www.codeproject.com/articles/716530/fastest-hash-function-for-table-lookups-in-c +static inline UINT32 FNV1A_Pippip_Yurii(UINT64* Address) +{ + const UINT32 PRIME = 591798841; UINT32 hash32; UINT64 hash64 = 14695981039346656037ULL; + hash64 = (hash64 ^ *Address) * PRIME; + hash32 = (uint32_t)(hash64 ^ (hash64 >> 32)); + return hash32 ^ (hash32 >> 16); +} + + +static void InitMemoryHash() +{ +} + +static void FreeMemoryHash() +{ + struct MemoryDataStruct* p, * tmp; + HASH_ITER(memory, MemoryHash, p, tmp) + { + HASH_DELETE(memory, MemoryHash, p); + free(p); + } +} + +static void UpdateMemoryData(UINT64* Location, UINT8 Offset, UINT64 WriteValue, UINT8 WriteSize) +{ + UINT64 OriginValue; + OriginValue = *Location; + switch (WriteSize) + { + case 1: // 1个字节 + Offset = Offset << 3; // 从字节换算成位 + OriginValue = (OriginValue & ~((UINT64)0xFF << Offset)) | ((WriteValue & (UINT64)0xFF) << Offset); + break; + case 2: // 2个字节 + Offset = Offset & 0xFE; // 对齐到2字节边界 + Offset = Offset << 3; // 从字节换算成位 + OriginValue = (OriginValue & ~((UINT64)0xFFFF << Offset)) | ((WriteValue & (UINT64)0xFFFF) << Offset); + break; + case 4: // 4个字节 + Offset = Offset & 0xFC; // 对齐到4字节边界 + Offset = Offset << 3; // 从字节换算成位 + OriginValue = (OriginValue & ~((UINT64)0xFFFFFFFF << Offset)) | ((WriteValue & (UINT64)0xFFFFFFFF) << Offset); + break; + case 8: // 8个字节 + OriginValue = WriteValue; + break; + } + *Location = OriginValue; +} + +static void WriteMemoryHash(UINT64 Address, UINT64 WriteValue, UINT8 WriteSize) +{ + UINT64 AlignAddress; + UINT8 Offset; + struct MemoryDataStruct* s; + + AlignAddress = Address & 0xFFFFFFFFFFFFFFF8; + Offset = Address & 0x7; + + HASH_FIND(memory, MemoryHash, &AlignAddress, sizeof(AlignAddress), s); + if (s == NULL) + { + // 没有在MemoryData中命中 + s = (struct MemoryDataStruct*)malloc(sizeof(struct MemoryDataStruct)); + s->Address = AlignAddress; + s->Data = 0xDEADBEEFDEADC0DE; + HASH_ADD(memory, MemoryHash, Address, sizeof(AlignAddress), s); + } + UpdateMemoryData(&(s->Data), Offset, WriteValue, WriteSize); +} + +static UINT64 ReadMemoryHash(UINT64 Address, UINT8 ReadSize) +{ + UINT64 AlignAddress; + UINT8 Offset; + struct MemoryDataStruct* s; + UINT64 OriginValue, ReadValue; + + AlignAddress = Address & 0xFFFFFFFFFFFFFFF8; + Offset = Address & 0x7; + + HASH_FIND(memory, MemoryHash, &AlignAddress, sizeof(AlignAddress), s); + if (s == NULL) + { + // 没有在MemoryData中命中，出错了！ + if (VERBOSE_MSG) + printf("[%s] 试图从一个未初始化的内存读取数据Address=%016llX！\n", __func__, Address); + OriginValue = 0xDEADBEEFDEADC0DE; + } + else + { + OriginValue = s->Data; + } + + /* + * 根据Offset和Size，对读出的数据进行修正，对齐到小端 + */ + ReadValue = OriginValue; + switch (ReadSize) + { + case 1: // 1个字节 + Offset = Offset << 3; // 从字节换算成位 + ReadValue = (OriginValue >> Offset) & 0xFF; + break; + case 2: // 2个字节 + Offset = Offset & 0xFE; // 对齐到2字节边界 + Offset = Offset << 3; // 从字节换算成位 + ReadValue = (OriginValue >> Offset) & 0xFFFF; + break; + case 4: // 4个字节 + Offset = Offset & 0xFC; // 对齐到4字节边界 + Offset = Offset << 3; // 从字节换算成位 + ReadValue = (OriginValue >> Offset) & 0xFFFFFFFF; + break; + case 8: // 8个字节 + ReadValue = OriginValue; + break; + + } + return ReadValue; +} + +UINT64 ReadMemory(UINT64 Address) +{ + GlobalSimReadMemoryCounter++; + return ReadMemoryHash(Address, 8); +} + +void WriteMemory(UINT64 Address, UINT64 WriteData) +{ + GlobalSimWriteMemoryCounter++; + WriteMemoryHash(Address, WriteData, 8); +} + +static void MemoryTraceStat() +{ + UINT64 i; + + GlobalMemoryInstCounter = 0; + GlobalMemoryReadCounter = 0; + GlobalMemoryWriteCounter = 0; + GlobalMemoryModifyCounter = 0; + + for (i = 0; i < MemoryTraceCounter; i++) + { + if (MemoryTrace[i].Operation == 'I') + GlobalMemoryInstCounter++; + else if (MemoryTrace[i].Operation == 'L') + GlobalMemoryReadCounter++; + else if (MemoryTrace[i].Operation == 'S') + GlobalMemoryWriteCounter++; + else if (MemoryTrace[i].Operation == 'M') + GlobalMemoryModifyCounter++; + } +} + +static int process_TRACE_line(char* linebuf, UINT64 lineno) +{ + int i; + int ret = 0; + UINT8 Operation; + UINT64 Address; + UINT8 Size; + + UINT64 RandValue64; + + // 删除行首、行尾空白回车等 + Trim(linebuf); + int linelen = (int)strlen(linebuf); + // 跳过空行 + if (linelen == 0) + return 0; + // 跳过超长的行 + if (linelen >= 100) + return 0; + // 跳过注释行 + if (linebuf[0] == '#' || linebuf[0] == '/' || linebuf[0] == '=' || linebuf[0] == '-') + return 0; + //printf("lineno=%llu\n", lineno); + char** pt; + int field_count; + pt = split_space(linebuf, &field_count); + + if (field_count == 3 || field_count == 4) + { + // 格式： Operation Address,Size + Operation = pt[0][0]; + if (field_count == 3) + { + Address = strtoull(pt[1], NULL, 16); + Size = atoi(pt[2]); + } + else + { + Address = strtoull(pt[2], NULL, 16); + Size = atoi(pt[3]); + } + + if (Operation != 'I' && Operation != 'L' && Operation != 'S' && Operation != 'M') + { + if (VERBOSE_MSG) + printf("[%s] line %lld 操作类型错误，不是I、L、S、M！ %s\n", __func__, lineno, linebuf); + ret = -1; + } + else if (Size != 1 && Size != 2 && Size != 4 && Size != 8) + { + if (VERBOSE_MSG) + printf("[%s] line %lld 数据大小错误，不是1、2、4、8！ %s\n", __func__, lineno, linebuf); + ret = -1; + } + else + { + RandValue64 = GetRand64(); + + // 初始化阶段，需要将所有地址的数据，设置为随机值 + WriteMemoryHash(Address, RandValue64, Size); + + if (MemoryTraceCounter >= MemoryTraceCapacity) + { + if (VERBOSE_MSG) + printf("[%s] line %lld 扩展内存到%llu！\n", __func__, lineno, MemoryTraceCapacity + MEMORY_TRACE_CHUNK); + MemoryTrace = (struct MemoryTraceStruct*)realloc(MemoryTrace, (MemoryTraceCapacity + MEMORY_TRACE_CHUNK) * sizeof(struct MemoryTraceStruct)); + if (MemoryTrace == NULL) + { + printf("[%s] line %lld 分配内存失败！\n", __func__, lineno); + ret = -1; + } + else + MemoryTraceCapacity += MEMORY_TRACE_CHUNK; + } + + if (MemoryTrace) + { + MemoryTrace[MemoryTraceCounter].Operation = Operation; + MemoryTrace[MemoryTraceCounter].Address = Address; + MemoryTrace[MemoryTraceCounter].Size = Size; + MemoryTraceCounter++; + + ret = 0; + } + } + } + else + { + printf("[%s] line %lld 格式错误！[fields=%d] %s\n", __func__, lineno, field_count, linebuf); + ret = -1; + } + + for (i = 0; i < field_count; i++) + free(pt[i]); + free(pt); + return ret; +} + +static int parse_TRACE_file(char* filename) +{ + UINT64 lineno; + int ret_parse_line = 0; + clock_t tick1, tick2; + cbsl_errors cbsl_ret = cbsl_error; + char linebuf[CHUNK]; + + tick1 = clock(); + cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, filename); + if (ctx == NULL) + { + printf("[%s] 不能以读方式打开Trace文件 %s\n", __func__, filename); + return -1; + } + + lineno = 0; + do + { + lineno++; + cbsl_ret = cbsl_readline(ctx, linebuf, sizeof(linebuf)); + CBSL_ERROR_CHECK(cbsl_ret); // 从文件中读取1行 + + ret_parse_line = process_TRACE_line(linebuf, lineno); // 分析处理1行 + if (ret_parse_line == 1 || cbsl_ret == cbsl_end) + break; + + if (lineno % 10000 == 0) + { + printf("\33[?25l[%s] ====已处理%llu行====\r", __func__, lineno); // 隐藏光标，显示进度 + } + } while (1); + + printf("\n\33[?25h"); // 显示光标 + CBSL_ERROR_CHECK(cbsl_close(ctx)); + + tick2 = clock(); + + MemoryTraceStat(); + printf("[%s] +-----------------------------------------------------+\n", __func__); + printf("[%s] | Memory Trace数量 \t : %10llu |\n", __func__, MemoryTraceCounter); + printf("[%s] | Instruction操作数量 \t : %10llu |\n", __func__, GlobalMemoryInstCounter); + printf("[%s] | Data Load操作数量 \t : %10llu |\n", __func__, GlobalMemoryReadCounter); + printf("[%s] | Data Store操作数量 \t : %10llu |\n", __func__, GlobalMemoryWriteCounter); + printf("[%s] | Data Modify操作数量 \t : %10llu |\n", __func__, GlobalMemoryModifyCounter); + printf("[%s] | 时间耗费（ms） \t : %10.0f |\n", __func__, ((float)(tick2 - tick1) / CLOCKS_PER_SEC) * 1000.0); + printf("[%s] +-----------------------------------------------------+\n", __func__); + + if (MemoryTraceCounter == 0) + return -1; + return 0; +} + +int SimTrace() +{ + int ret = 0; + UINT32 i; + UINT8 Operation; + UINT64 Address; + UINT8 Size; + + UINT64 RandValue64; + UINT64 DataFromCache; + UINT64 DataFromMemory; + UINT8 MissFlag; + + clock_t tick1, tick2; + tick1 = clock(); + + GlobalMemoryInstCounter = 0; + GlobalMemoryReadCounter = 0; + GlobalMemoryWriteCounter = 0; + GlobalMemoryModifyCounter = 0; + + GlobalCacheInstHitCounter = 0; + GlobalCacheReadHitCounter = 0; + GlobalCacheWriteHitCounter = 0; + GlobalCacheModifyHitCounter = 0; + + GlobalSimReadMemoryCounter = 0; + GlobalSimWriteMemoryCounter = 0; + + for (i = 0; i < MemoryTraceCounter; i++) + { + Operation = MemoryTrace[i].Operation; + Address = MemoryTrace[i].Address; + Size = MemoryTrace[i].Size; + + if (Operation == 'L' || Operation == 'S' || Operation == 'M') + { + RandValue64 = GetRand64(); + MissFlag = AccessDataCache(Address, Operation, Size, RandValue64, &DataFromCache); + if (Operation == 'S' || Operation == 'M') + { + WriteMemoryHash(Address, RandValue64, Size); + if (Operation == 'S') + { + GlobalMemoryWriteCounter++; + if (MissFlag == 'H') + { + GlobalCacheWriteHitCounter++; + } + } + else if (Operation == 'M') + { + GlobalMemoryModifyCounter++; + if (MissFlag == 'H') + { + GlobalCacheModifyHitCounter++; + } + } + } + else if (Operation == 'L') + { + GlobalMemoryReadCounter++; + if (MissFlag == 'H') + { + DataFromMemory = ReadMemoryHash(Address, Size); + if (DataFromMemory == DataFromCache) + GlobalCacheReadHitCounter++; + else + { + printf("[%s] 关键错误！数据Cache读错误，内存地址=%016llX 内存数据=%016llX Cache读数据=%016llX 大小%d字节\n", __func__, Address, DataFromMemory, DataFromCache, Size); + ret = -1; + break; + } + } + } + } + else if (Operation == 'I') + { + MissFlag = AccessInstCache(Address, Operation, Size, &DataFromCache); + GlobalMemoryInstCounter++; + if (MissFlag == 'H') + { + DataFromMemory = ReadMemoryHash(Address, Size); + if (DataFromMemory == DataFromCache) + GlobalCacheInstHitCounter++; + else + { + printf("[%s] 关键错误！指令Cache读错误，内存地址=%016llX 内存数据=%016llX Cache读数据=%016llX 大小%d字节\n", __func__, Address, DataFromMemory, DataFromCache, Size); + ret = -1; + break; + } + } + } + } + tick2 = clock(); + printf("[%s] +-----------------------------------------------------+\n", __func__); + printf("[%s] | Memory Trace数量 \t : %10llu |\n", __func__, MemoryTraceCounter); + printf("[%s] | Instruction操作数量 \t : %10llu |\n", __func__, GlobalMemoryInstCounter); + printf("[%s] | Data Load操作数量 \t : %10llu |\n", __func__, GlobalMemoryReadCounter); + printf("[%s] | Data Store操作数量 \t : %10llu |\n", __func__, GlobalMemoryWriteCounter); + printf("[%s] | Data Modify操作数量 \t : %10llu |\n", __func__, GlobalMemoryModifyCounter); + printf("[%s] | Instruction操作Cache命中数量 \t : %10llu |\n", __func__, GlobalCacheInstHitCounter); + printf("[%s] | Data Load操作Cache命中数量 \t : %10llu |\n", __func__, GlobalCacheReadHitCounter); + printf("[%s] | Data Store操作Cache命中数量 \t : %10llu |\n", __func__, GlobalCacheWriteHitCounter); + printf("[%s] | Data Modify操作Cache命中数量 \t : %10llu |\n", __func__, GlobalCacheModifyHitCounter); + printf("[%s] | Cache访存数量 \t : %10llu |\n", __func__, GlobalSimReadMemoryCounter + GlobalSimWriteMemoryCounter); + printf("[%s] | Cache读存储器数量 \t : %10llu |\n", __func__, GlobalSimReadMemoryCounter); + printf("[%s] | Cache写存储器数量 \t : %10llu |\n", __func__, GlobalSimWriteMemoryCounter); + printf("[%s] | Data Cache命中率 \t : %9.2f%% |\n", __func__, (double)((GlobalCacheReadHitCounter + GlobalCacheWriteHitCounter + GlobalCacheModifyHitCounter) * 100) / (double)(GlobalMemoryReadCounter + GlobalMemoryWriteCounter + GlobalMemoryModifyCounter)); + printf("[%s] | Inst Cache命中率 \t : %9.2f%% |\n", __func__, (GlobalMemoryInstCounter == 0) ? 0 : (double)(GlobalCacheInstHitCounter * 100) / (double)GlobalMemoryInstCounter); + printf("[%s] | 时间耗费（ms） \t : %10.0f |\n", __func__, ((float)(tick2 - tick1) / CLOCKS_PER_SEC) * 1000.0); + printf("[%s] +-----------------------------------------------------+\n", __func__); + return ret; +} +void DisplayHelp(char* argv[]) +{ + printf("[%s] 请在Cache.c中，实现你自己的Cache，然后编译项目，执行。\n", __func__); + printf("[%s] 从文本格式压缩文件中读取Trace:\t%s .zst\n", __func__, argv[0]); + printf("[%s] 例如: %s ./traces/dave.trace.zst\n", __func__, argv[0]); + printf("[%s] 将文本格式的Trace转换到bin格式:\t%s -w .zst\n", __func__, argv[0]); + printf("[%s] 从bin格式文件中读取Trace:\t\t%s -r .bin.zst\n", __func__, argv[0]); + printf("[%s] 提示：从bin中读取Trace速度要远远快于从文本格式中读取。\n", __func__); +} + +int main(int argc, char* argv[]) +{ + int ret_val = -1; + UINT8 ReadBinFileFlag, WriteBinFileFlag, ReadTxtFileFlag; + char* pfilename; + char pfilename_bin[CHUNK]; + clock_t tick1, tick2; + UINT64 i; + + printf("[%s] Cache模拟器框架 v3.0 by mars, 2022\n", __func__); + + pfilename = NULL; + ReadBinFileFlag = 0; + WriteBinFileFlag = 0; + ReadTxtFileFlag = 1; + /* check arguments */ + while (1) { + int c = getopt(argc, argv, "-hrw"); + if (c == -1) break; + + switch (c) { + case 'h': DisplayHelp(argv); return 1; + case 'r': ReadBinFileFlag = 1; ReadTxtFileFlag = 0; break; + case 'w': WriteBinFileFlag = 1; ReadTxtFileFlag = 0; break; + case 1: pfilename = optarg; break; + } + } + + if ((ReadBinFileFlag == 1 || WriteBinFileFlag == 1 || ReadTxtFileFlag == 1) && pfilename == NULL) + { + DisplayHelp(argv); + return 1; + } + + InitMemoryHash(); + printf("[%s] 初始化存储器，读入Trace文件[%s]，请稍后...\n", __func__, pfilename); + + if (ReadTxtFileFlag || WriteBinFileFlag) + { + MemoryTrace = (struct MemoryTraceStruct*)malloc(MEMORY_TRACE_CHUNK * sizeof(struct MemoryTraceStruct)); + MemoryTraceCounter = 0; + MemoryTraceCapacity = MEMORY_TRACE_CHUNK; + ret_val = parse_TRACE_file(pfilename); + if (ret_val != 0) + { + FreeMemoryHash(); + if (MemoryTrace) + free(MemoryTrace); + printf("[%s] 解压缩文件失败 %s\n", __func__, argv[1]); + return -1; + } + + if (WriteBinFileFlag) + { + // 将内存中的MemoryTrace保存到bin文件中 + int filenamelen = (int)strlen(pfilename); + memcpy(pfilename_bin, pfilename, filenamelen); + pfilename_bin[filenamelen] = '\0'; + if (filenamelen > 5) + { + if ((pfilename_bin[filenamelen - 4] == '.' && pfilename_bin[filenamelen - 3] == 'z' && pfilename_bin[filenamelen - 2] == 's' && pfilename_bin[filenamelen - 1] == 't') || + (pfilename_bin[filenamelen - 4] == '.' && pfilename_bin[filenamelen - 3] == 'Z' && pfilename_bin[filenamelen - 2] == 'S' && pfilename_bin[filenamelen - 1] == 'T')) + { + pfilename_bin[filenamelen - 3] = 'b'; + pfilename_bin[filenamelen - 2] = 'i'; + pfilename_bin[filenamelen - 1] = 'n'; + pfilename_bin[filenamelen - 0] = '.'; + pfilename_bin[filenamelen + 1] = 'z'; + pfilename_bin[filenamelen + 2] = 's'; + pfilename_bin[filenamelen + 3] = 't'; + pfilename_bin[filenamelen + 4] = '\0'; + + cbsl_ctx* ctx = cbsl_open(cbsl_store_mode, pfilename_bin); + if (ctx == NULL) + { + printf("[%s] 不能以写方式打开文件 %s\n", __func__, pfilename_bin); + ret_val = -1; + } + else + { + CBSL_ERROR_CHECK(cbsl_write(ctx, &MemoryTraceCounter, sizeof(MemoryTraceCounter))); + CBSL_ERROR_CHECK(cbsl_write(ctx, MemoryTrace, MemoryTraceCounter * sizeof(struct MemoryTraceStruct))); + CBSL_ERROR_CHECK(cbsl_close(ctx)); + printf("[%s] 已经将Trace保存到文件中 %s\n", __func__, pfilename_bin); + ret_val = 0; + } + } + else + { + printf("[%s] 文件扩展名不是.zst或者.ZST！不能转换！\n", __func__); + ret_val = -1; + } + } + else + { + printf("[%s] 文件名长度不足5字符！不能转换！\n", __func__); + ret_val = -1; + } + + FreeMemoryHash(); + if (MemoryTrace) + free(MemoryTrace); + return ret_val; + } + } + else if (ReadBinFileFlag) + { + cbsl_ctx* ctx = cbsl_open(cbsl_load_mode, pfilename); + if (ctx == NULL) + { + printf("[%s] 不能以读方式打开文件 %s\n", __func__, pfilename); + ret_val = -1; + } + else + { + tick1 = clock(); + CBSL_ERROR_CHECK(cbsl_read(ctx, &MemoryTraceCounter, sizeof(MemoryTraceCounter))); + { + MemoryTrace = (struct MemoryTraceStruct*)malloc(MemoryTraceCounter * sizeof(struct MemoryTraceStruct)); + MemoryTraceCapacity = MemoryTraceCounter; + + CBSL_ERROR_CHECK(cbsl_read(ctx, MemoryTrace, MemoryTraceCounter * sizeof(struct MemoryTraceStruct))); + CBSL_ERROR_CHECK(cbsl_close(ctx)); + { + // 初始化阶段，需要将所有地址的数据，设置为随机值 + UINT64 RandValue64; + for (i = 0; i < MemoryTraceCounter; i++) + { + RandValue64 = GetRand64(); + WriteMemoryHash(MemoryTrace[i].Address, RandValue64, MemoryTrace[i].Size); + } + + MemoryTraceStat(); + tick2 = clock(); + printf("[%s] +-----------------------------------------------------+\n", __func__); + printf("[%s] | Memory Trace数量 \t : %10llu |\n", __func__, MemoryTraceCounter); + printf("[%s] | Instruction操作数量 \t : %10llu |\n", __func__, GlobalMemoryInstCounter); + printf("[%s] | Data Load操作数量 \t : %10llu |\n", __func__, GlobalMemoryReadCounter); + printf("[%s] | Data Store操作数量 \t : %10llu |\n", __func__, GlobalMemoryWriteCounter); + printf("[%s] | Data Modify操作数量 \t : %10llu |\n", __func__, GlobalMemoryModifyCounter); + printf("[%s] | 时间耗费（ms） \t : %10.0f |\n", __func__, ((float)(tick2 - tick1) / CLOCKS_PER_SEC) * 1000.0); + printf("[%s] +-----------------------------------------------------+\n", __func__); + printf("[%s] 已经将Trace从文件中读取 %s\n", __func__, pfilename); + ret_val = 0; + } + } + } + + if (ret_val == -1) + { + if (MemoryTrace) + free(MemoryTrace); + FreeMemoryHash(); + return ret_val; + } + } + printf("[%s] 处理Trace文件完毕\n", __func__); + InitDataCache(); + InitInstCache(); + printf("[%s] 开始Cache模拟，请稍后...\n", __func__); + ret_val = SimTrace(); + if (ret_val != 0) + { + FreeMemoryHash(); + if (MemoryTrace) + free(MemoryTrace); + printf("[%s] Cache模拟失败\n", __func__); + return -1; + } + else + printf("[%s] Cache模拟成功完成\n", __func__); + if (MemoryTrace) + free(MemoryTrace); + FreeMemoryHash(); + return 0; +} diff --git a/cachelab/Makefile b/cachelab/Makefile new file mode 100644 index 0000000..799f83a --- /dev/null +++ b/cachelab/Makefile @@ -0,0 +1,26 @@ +# Copyright 2022 by mars + +# Description: Makefile for building a Cache Simulator. +# + + +LDFLAGS += + +LDLIBS += -lzstd + +CPPFLAGS := -O3 -Wall -Wextra -Winline -Winit-self -Wno-sequence-point\ + -Wno-unused-function -Wno-inline -fPIC -W -Wcast-qual -Wpointer-arith -Icbsl/include + +#CPPFLAGS := -g +PROGRAMS := Cache + +objects = Cache.o CacheHelper.o getopt.o cbsl/src/buffer.o cbsl/src/file.o cbsl/src/flush.o cbsl/src/read.o cbsl/src/record.o cbsl/src/utils.o cbsl/src/write.o + +all: $(PROGRAMS) + +Cache : $(objects) + gcc $(CPPFLAGS) $(LDFLAGS) -o $@ $^ $(LDLIBS) + rm -f $(objects) + +clean: + rm -f $(PROGRAMS) $(objects) diff --git a/cachelab/MyCache/MyCache.sln b/cachelab/MyCache/MyCache.sln new file mode 100644 index 0000000..e164b0f --- /dev/null +++ b/cachelab/MyCache/MyCache.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.31829.152 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MyCache", "MyCache.vcxproj", "{A5677060-D4C9-432A-A29F-858971B64066}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A5677060-D4C9-432A-A29F-858971B64066}.Debug|x64.ActiveCfg = Debug|x64 + {A5677060-D4C9-432A-A29F-858971B64066}.Debug|x64.Build.0 = Debug|x64 + {A5677060-D4C9-432A-A29F-858971B64066}.Debug|x86.ActiveCfg = Debug|Win32 + {A5677060-D4C9-432A-A29F-858971B64066}.Debug|x86.Build.0 = Debug|Win32 + {A5677060-D4C9-432A-A29F-858971B64066}.Release|x64.ActiveCfg = Release|x64 + {A5677060-D4C9-432A-A29F-858971B64066}.Release|x64.Build.0 = Release|x64 + {A5677060-D4C9-432A-A29F-858971B64066}.Release|x86.ActiveCfg = Release|Win32 + {A5677060-D4C9-432A-A29F-858971B64066}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {D4625C4E-654D-406B-A238-D6F64DAF03BB} + EndGlobalSection +EndGlobal diff --git a/cachelab/MyCache/MyCache.vcxproj b/cachelab/MyCache/MyCache.vcxproj new file mode 100644 index 0000000..9fde92b --- /dev/null +++ b/cachelab/MyCache/MyCache.vcxproj @@ -0,0 +1,158 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 16.0 + Win32Proj + {a5677060-d4c9-432a-a29f-858971b64066} + MyCache + 10.0 + + + + Application + true + v142 + Unicode + + + Application + false + v142 + true + Unicode + + + Application + true + v142 + Unicode + + + Application + false + v142 + true + Unicode + + + + + + + + + + + + + + + + + + + + + true + + + false + + + true + + + false + + + + Level3 + true + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + + + + + Level3 + true + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + + + Console + true + true + true + + + + + Level3 + true + _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + ..\zstd;..\cbsl\include;%(AdditionalIncludeDirectories) + + + Console + true + + + + + Level3 + true + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + true + ..\zstd;..\cbsl\include;%(AdditionalIncludeDirectories) + + + Console + true + true + true + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/cachelab/Readme.txt b/cachelab/Readme.txt new file mode 100644 index 0000000..9c7dd75 --- /dev/null +++ b/cachelab/Readme.txt @@ -0,0 +1,26 @@ +注意！你只能修改Cache.c文件，其他文件请勿修改！ + + +Linux: + 1. make + 2. ./Cache traces/long.trace.zst + +提醒:Linux下需要安装libzstd-dev软件包，例如apt install libzstd-dev zstd + +Windows VS 2019: + 1. 进入MyCache目录，打开MyCache.sln + 2. 编译即可生成可执行文件 + 3. 启动命令行窗口， 运行.\MyCache.exe traces/long.trace.zst + + +提示：在traces目录下，有多个trace可以跑。 + + +自行生成trace文件： +1、安装valgrind，Ubuntu下，可以使用apt install valgrind完成； +2、运行valgrind生成某个应用的trace文件。比如，为了生成ls命令运行时的trace，可以输入下列命令： +valgrind --tool=lackey -v --trace-mem=yes --log-file=ls.trace ls -l +3、将生成的trace文件打包为zst压缩格式，例如 + zstd ls.trace -o ls.trace.zst +4、使用Cache跑这个trace： + ./Cache ./ls.trace.zst diff --git a/cachelab/cbsl/CMakeLists.txt b/cachelab/cbsl/CMakeLists.txt new file mode 100644 index 0000000..6676f48 --- /dev/null +++ b/cachelab/cbsl/CMakeLists.txt @@ -0,0 +1,125 @@ +# +# Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +cmake_minimum_required(VERSION 3.3) + +enable_testing() + +set(CMAKE_C_STANDARD 99) + +project(cbsl + VERSION 2019.5.0 + LANGUAGES C Fortran + ) + + +if (CMAKE_BUILD_TYPE) + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if (NOT uppercase_CMAKE_BUILD_TYPE MATCHES "^(DEBUG|RELEASE|RELWITHDEBINFO|MINSIZEREL)$") + message(FATAL_ERROR "Invalid value for CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") + endif() + + if (${uppercase_CMAKE_BUILD_TYPE} MATCHES "DEBUG") + set(CBSL_DEBUG 1) + endif () +endif () + + +if (INSTALL_ZSTD) + message(STATUS "Enable installation Zstandard library version 1.4.0") + include(ExternalProject) + ExternalProject_Add(zstd + GIT_REPOSITORY "https://github.com/facebook/zstd.git" + GIT_TAG "v1.4.0" + PREFIX "${CMAKE_BINARY_DIR}/zstd" + SOURCE_SUBDIR "build/cmake" + CMAKE_ARGS -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -D CMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} + STEP_TARGETS install + EXCLUDE_FROM_ALL on + ) + include_directories(${CMAKE_INSTALL_PREFIX}/include) + link_directories(${CMAKE_INSTALL_PREFIX}/lib) +endif () +find_library(ZSTD_LIB NAMES "zstd") + + +include(CheckCCompilerFlag) +CHECK_C_COMPILER_FLAG("-Wall" C_HAS_WALL) +CHECK_C_COMPILER_FLAG("-Wshadow" C_HAS_WSHADOW) +CHECK_C_COMPILER_FLAG("-Werror" C_HAS_WERROR) +CHECK_C_COMPILER_FLAG("-pedantic-errors" C_HAS_PEDANTIC_ERRORS) + +set(ADD_C_EXTRA_FLAGS) + +if (C_HAS_WALL) + set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -Wall") +endif () + +if (C_HAS_WSHADOW) + set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -Wshadow") +endif () + +if (C_HAS_WERROR) + set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -Werror") +endif () + +if (C_HAS_PEDANTIC_ERRORS) + set(ADD_C_EXTRA_FLAGS "${ADD_C_EXTRA_FLAGS} -pedantic-errors") +endif () + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ADD_C_EXTRA_FLAGS}") + + +include(CheckFortranCompilerFlag) +CHECK_Fortran_COMPILER_FLAG("-Wall -Wextra" Fortran_HAS_WALL_EXTRA) +CHECK_Fortran_COMPILER_FLAG("-Wshadow" Fortran_HAS_WERROR) +CHECK_Fortran_COMPILER_FLAG("-pedantic-errors" Fortran_HAS_PEDANTIC_ERRORS) + +set(ADD_Fortran_EXTRA_FLAGS) + +if (Fortran_HAS_WALL_EXTRA) + set(ADD_Fortran_EXTRA_FLAGS "${ADD_Fortran_EXTRA_FLAGS} -Wall -Wextra") +endif () + +if (Fortran_HAS_WERROR) + set(ADD_Fortran_EXTRA_FLAGS "${ADD_Fortran_EXTRA_FLAGS} -Werror") +endif () + +if (Fortran_HAS_PEDANTIC_ERRORS) + set(ADD_Fortran_EXTRA_FLAGS "${ADD_Fortran_EXTRA_FLAGS} -pedantic-errors") +endif () + +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${ADD_Fortran_EXTRA_FLAGS}") + + +include_directories(${PROJECT_SOURCE_DIR}/include) +include_directories(${PROJECT_BINARY_DIR}/include) +set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/include) + + +set(CBSL_LIB "cbsl") +set(CBSL_FLIB "${CBSL_LIB}f") + +add_subdirectory(include) +add_subdirectory(src) +add_subdirectory(examples) +add_subdirectory(tests) +add_subdirectory(benchmarks) + +if (INSTALL_ZSTD) + # install zstd before building the library + add_dependencies(${CBSL_FLIB} ${CBSL_LIB}) + add_dependencies(${CBSL_LIB} zstd-install) +endif () diff --git a/cachelab/cbsl/LICENSE b/cachelab/cbsl/LICENSE new file mode 100644 index 0000000..41ce108 --- /dev/null +++ b/cachelab/cbsl/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Yuta Hirokawa (University of Tsukuba, Japan) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cachelab/cbsl/README.md b/cachelab/cbsl/README.md new file mode 100644 index 0000000..3d7394d --- /dev/null +++ b/cachelab/cbsl/README.md @@ -0,0 +1,93 @@ +# CBSL: Compressed Binary Serialization Library + +## What's this? + +The library provides the binary serialization with the compression by [Zstandard](https://facebook.github.io/zstd/). +A motivation of the library is to implement the data-compressed checkpoint/restart, which is well-known technique to recover computer failures in high-performance computing. +This library aims to simple and lightweight access for users. + +The library support C99 or later, and Fortran 2008 or later: perhaps we use the features are all supported by major compilers. + +## How to build and test + +The library uses [CMake](https://cmake.org/) version 3.3.x or later. + + $ cmake --version + cmake version 3.14.3 + + CMake suite maintained and supported by Kitware (kitware.com/cmake). + $ mkdir build && cd build + $ cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/install/path .. + $ make + $ make test + $ make install + +## Do you need help to install `zstd` package? + +We can build and install `zstd` automatically in the build process. +Please pass `-D INSTALL_ZSTD=on` to cmake, we will install `zstd` package where `CMAKE_INSTALL_PREFIX` directory before building the library. + + $ cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/install/path -D INSTALL_ZSTD=on .. + $ make + $ make test + $ make install + +## Test environments + +1. CMake version 3.14.3 +2. GCC version 4.8.5 +3. Zstandard version 1.4.0 +4. CentOS Linux release 7.5.1804 (Core) + +## Benchmark + +We provide the library performance benchmark with best and worst case. +Please be reminded that **the benchmark results not indicate the performance of Zstandard**, these measure the overhead (use cost) of this library. + +1. Best case : all data is zero filled (A compression ratio achieves up to 99%) +2. Worst case : data is generated by rand() (A compression ratio is lower than 1%) + +`benchmark` target executes the benchmarks. + + ... + $ make benchmark + min data size = 262144.00 [B] + max data size = 134217728.00 [B] + data is zero filled (maximum compression) +