test_csv source code

1 /** Helper to read and write CSV files */
2 module test_csv;
3 
4 import std.datetime;
5 import std.digest;
6 import std.digest.md;
7 version (UseXXHSourceFromDub) import xxhash3;
8 version (UseXXHSourceFromLocalDir) import local_dev.xxh;
9 version (UseXXHSourceFromPhobos) import std.digest.xxh;
10 import std.digest.murmurhash;
11 import std.file;
12 import std.format;
13 import std.range : join;
14 import std.stdio;
15 import std.string;
16 import std.traits : FieldNameTuple;
17 
18 import test_helpers;
19 import test_pattern_generator;
20 
21 /** Container struct for compression benchmarking in CSV file */
22 struct CompressionTestData {
23 public:
24     TestPattern pattern;
25     size_t unpacked_size;
26     size_t packed_size;
27     Duration pack_duration;
28     Duration unpack_duration;
29 private:
30     float pack_ratio;
31     float pack_mbs;
32     float unpack_mbs;
33 
34     void updateHiddenFields() {
35         pack_ratio = getCompressionRatio(unpacked_size, packed_size);
36         pack_mbs = getMegaBytePerSeconds(unpacked_size, pack_duration);
37         unpack_mbs = getMegaBytePerSeconds(unpacked_size, unpack_duration);
38     }
39 
40 public:
41     /** Get the names of the CSV fields from the structure */
42     static string getCsvHeader() {
43         return [FieldNameTuple!CompressionTestData].join(';');
44     }
45 
46     /** Output the structure data as a CSV Line */
47     string getCsvLine() {
48         updateHiddenFields();
49         auto rc = format("%s;%d;%d;%f;%f;%f;%f;%f",
50             pattern, unpacked_size, packed_size,
51             getFloatSecond(pack_duration), getFloatSecond(unpack_duration),
52             pack_ratio, pack_mbs, unpack_mbs
53         );
54         return rc;
55     }
56 }
57 
58 /** Check empty values and zero devision handling */
59 unittest {
60     CompressionTestData tv;
61     const auto hdr = tv.getCsvHeader;
62     assert(
63         hdr == "pattern;unpacked_size;packed_size;pack_duration;unpack_duration;pack_ratio;pack_mbs;unpack_mbs");
64     const auto line = tv.getCsvLine;
65     assert(line == "(Unknown:00000:00000:00);0;0;0.000000;0.000000;nan;nan;nan");
66 }
67 
68 /** Check example values*/
69 unittest {
70     auto tp = TestPattern(PatterMode.RepeatN, 0x00100, 0x00200, 8);
71 
72     CompressionTestData tv = CompressionTestData(tp, 2 ^^ 20, 2 ^^ 10, dur!"seconds"(1), dur!"seconds"(
73             1));
74     auto line = tv.getCsvLine;
75     assert(
76         line == "(RepeatN:00100:00200:08);1048576;1024;1.000000;1.000000;0.000977;1.000000;1.000000");
77 
78     tv = CompressionTestData(tp, 10 ^^ 6, 10 ^^ 3, dur!"seconds"(1), dur!"seconds"(1));
79     line = tv.getCsvLine;
80     assert(
81         line == "(RepeatN:00100:00200:08);1000000;1000;1.000000;1.000000;0.001000;0.953674;0.953674");
82 }
83 
84 /** Container struct for hashing benchmarking in CSV file */
85 struct HashTestData(HASHTYPE) {
86 public:
87     size_t size;
88     size_t fill;
89     HASHTYPE hash;
90     Duration duration1;
91     Duration duration2;
92 private:
93     float mbs1;
94     float mbs2;
95 
96     void updateHiddenFields() {
97         mbs1 = getMegaBytePerSeconds(size, duration1);
98         mbs2 = getMegaBytePerSeconds(size, duration2);
99     }
100 
101 public:
102     /** Get the names of the CSV fields from the structure */
103     static string getCsvHeader() {
104         return [FieldNameTuple!MD5TestData].join(';');
105     }
106 
107     /** Output the structure data as a CSV Line */
108     string getCsvLine() {
109         import std.digest : toHexString;
110 
111         updateHiddenFields();
112         static if (is(HASHTYPE == MurmurHash3!32)) {
113             auto rc = format("0x%04x;0x%02x;%s;%f;%f;%f;%f", size, fill, hash.getBytes.toHexString,
114                 getFloatSecond(duration1), getFloatSecond(duration2),
115                 mbs2, mbs2);
116         } else static if (is(HASHTYPE == MurmurHash3!128)) {
117             auto rc = format("0x%04x;0x%02x;%s;%f;%f;%f;%f", size, fill, hash.getBytes.toHexString,
118                 getFloatSecond(duration1), getFloatSecond(duration2),
119                 mbs2, mbs2);
120         } else {
121             auto rc = format("0x%04x;0x%02x;%s;%f;%f;%f;%f", size, fill, hash.toHexString,
122                 getFloatSecond(duration1), getFloatSecond(duration2),
123                 mbs2, mbs2);
124         }
125         return rc;
126     }
127 }
128 
129 alias CRC32TestData = HashTestData!(ubyte[4]);
130 alias CRC64TestData = HashTestData!(ubyte[8]);
131 alias MD5TestData = HashTestData!(ubyte[16]);
132 alias XXH32TestData = HashTestData!XXH32_canonical_t;
133 alias XXH64TestData = HashTestData!XXH64_canonical_t;
134 alias XXH3_64TestData = HashTestData!XXH64_canonical_t;
135 alias XXH3_128TestData = HashTestData!XXH128_canonical_t;
136 alias Murmur32TestData = HashTestData!(ubyte[4]);
137 alias Murmur128TestData = HashTestData!(ubyte[16]);
138 
139 /** Check empty values and zero devision handling */
140 unittest {
141     MD5TestData tv;
142     const auto hdr = tv.getCsvHeader;
143     assert(hdr == "size;fill;hash;duration1;duration2;mbs1;mbs2");
144     const auto line = tv.getCsvLine;
145     assert(line == "0x0000;0x00;00000000000000000000000000000000;0.000000;0.000000;nan;nan");
146 }
147 
148 /** Check example values*/
149 unittest {
150     MD5TestData tv =
151         MD5TestData(
152             0x1000, 0xAA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
153             dur!"seconds"(1), dur!"seconds"(1));
154     auto line = tv.getCsvLine;
155     assert(
156         line == "0x1000;0xaa;000102030405060708090A0B0C0D0E0F;1.000000;1.000000;0.003906;0.003906");
157 
158     tv = MD5TestData(
159         0x2000, 0x55, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
160         dur!"seconds"(1), dur!"seconds"(1));
161     line = tv.getCsvLine;
162     assert(
163         line == "0x2000;0x55;000102030405060708090A0B0C0D0E0F;1.000000;1.000000;0.007812;0.007812");
164 }
165 
166 /** Check empty values and zero devision handling */
167 unittest {
168     XXH32TestData tv;
169     const auto hdr = tv.getCsvHeader;
170     assert(hdr == "size;fill;hash;duration1;duration2;mbs1;mbs2");
171     const auto line = tv.getCsvLine;
172     assert(line == "0x0000;0x00;00000000;0.000000;0.000000;nan;nan");
173 }
174 
175 /** Check example values*/
176 unittest {
177     XXH32TestData tv =
178         XXH32TestData(
179             0x1000, 0xAA, [ 12,34,56,78 ],
180             dur!"seconds"(1), dur!"seconds"(1));
181     auto line = tv.getCsvLine;
182     assert(line == "0x1000;0xaa;0C22384E;1.000000;1.000000;0.003906;0.003906");
183 
184     tv = XXH32TestData(
185         0x2000, 0x55, [ 12,34,56,78 ],
186         dur!"seconds"(1), dur!"seconds"(1));
187     line = tv.getCsvLine;
188     assert(line == "0x2000;0x55;0C22384E;1.000000;1.000000;0.007812;0.007812");
189 }
190 
191 /** Check empty values and zero devision handling */
192 unittest {
193     XXH64TestData tv;
194     const auto hdr = tv.getCsvHeader;
195     assert(hdr == "size;fill;hash;duration1;duration2;mbs1;mbs2");
196     const auto line = tv.getCsvLine;
197     assert(line == "0x0000;0x00;0000000000000000;0.000000;0.000000;nan;nan");
198 }
199 
200 /** Check example values*/
201 unittest {
202     XXH64TestData tv =
203         XXH64TestData(
204             0x1000, 0xAA, [ 1,2,3,4,5,6,7,8 ],
205             dur!"seconds"(1), dur!"seconds"(1));
206     auto line = tv.getCsvLine;
207     assert(line == "0x1000;0xaa;0102030405060708;1.000000;1.000000;0.003906;0.003906");
208 
209     tv = XXH64TestData(
210         0x2000, 0x55, [ 1,2,3,4,5,6,7,8 ],
211         dur!"seconds"(1), dur!"seconds"(1));
212     line = tv.getCsvLine;
213     assert(line == "0x2000;0x55;0102030405060708;1.000000;1.000000;0.007812;0.007812");
214 }
215 
216 /** Simple class to collect CSV data and write it to disk */
217 class TestCSVFile(T) {
218 private:
219     T[] testEntries; /// The collected test entries
220     string fileName; /// Name of CSV
221 
222 public:
223     /** Constructors */
224     this(string filename) {
225         fileName = filename;
226     }
227 
228     /** Add a test entry */
229     void addEntry(ref T entry) {
230         testEntries ~= entry;
231     }
232 
233     /** Write data as a CSV file */
234     void writeFile() {
235         /* Convert data to CSV strings */
236         string[] csvlines;
237         csvlines ~= T.getCsvHeader;
238         foreach (entry; testEntries)
239             csvlines ~= entry.getCsvLine;
240         auto csvblob = csvlines.join("\n") ~ "\n\n";
241         /* And finally write them to disk */
242         std.file.write(fileName, csvblob);
243     }
244 }
245 
246 unittest {
247     auto fn = deleteme();
248     auto tcf = new TestCSVFile!CompressionTestData(fn);
249     scope (exit)
250         remove(fn);
251     auto tc = CompressionTestData();
252     tcf.addEntry(tc);
253     tcf.writeFile;
254 
255     enum expectedContents =
256         "pattern;unpacked_size;packed_size;pack_duration;unpack_duration;pack_ratio;pack_mbs;unpack_mbs\n" ~
257         "(Unknown:00000:00000:00);0;0;0.000000;0.000000;nan;nan;nan\n\n";
258     const auto csvcontents = readText(fn);
259     assert(csvcontents == expectedContents, "Mismatched file contents");
260 }
261 
262 unittest {
263     auto tp = TestPattern(PatterMode.RepeatN, 0x00100, 0x00200, 8);
264 
265     auto fn = deleteme();
266     auto tcf = new TestCSVFile!CompressionTestData(fn);
267     scope (exit)
268         remove(fn);
269     auto tc1 = CompressionTestData(tp, 2 ^^ 20, 2 ^^ 10, dur!"seconds"(1), dur!"seconds"(1));
270     tcf.addEntry(tc1);
271     auto tc2 = CompressionTestData(tp, 10 ^^ 6, 10 ^^ 3, dur!"seconds"(1), dur!"seconds"(1));
272     tcf.addEntry(tc2);
273     tcf.writeFile;
274 
275     enum expectedContents =
276         "pattern;unpacked_size;packed_size;pack_duration;unpack_duration;pack_ratio;pack_mbs;unpack_mbs\n" ~
277         "(RepeatN:00100:00200:08);1048576;1024;1.000000;1.000000;0.000977;1.000000;1.000000\n" ~
278         "(RepeatN:00100:00200:08);1000000;1000;1.000000;1.000000;0.001000;0.953674;0.953674\n" ~
279         "\n";
280     const auto csvcontents = readText(fn);
281     assert(csvcontents == expectedContents, "Mismatched file contents");
282 }
283 
284 unittest {
285     auto fn = deleteme();
286     auto tcf = new TestCSVFile!MD5TestData(fn);
287     scope (exit)
288         remove(fn);
289     auto tc = MD5TestData();
290     tcf.addEntry(tc);
291     tcf.writeFile;
292 
293     enum expectedContents =
294         "size;fill;hash;duration1;duration2;mbs1;mbs2\n" ~
295         "0x0000;0x00;00000000000000000000000000000000;0.000000;0.000000;nan;nan\n\n";
296     const auto csvcontents = readText(fn);
297     assert(csvcontents == expectedContents, "Mismatched file contents");
298 }
299 
300 unittest {
301     auto fn = deleteme();
302     auto tcf = new TestCSVFile!MD5TestData(fn);
303     scope (exit)
304         remove(fn);
305     auto tc1 = MD5TestData(
306         0x1000, 0xAA, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
307         dur!"seconds"(1), dur!"seconds"(1));
308     tcf.addEntry(tc1);
309     auto tc2 = MD5TestData(
310         0x2000, 0x55, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
311         dur!"seconds"(1), dur!"seconds"(1));
312     tcf.addEntry(tc2);
313     tcf.writeFile;
314 
315     enum expectedContents =
316         "size;fill;hash;duration1;duration2;mbs1;mbs2\n" ~
317         "0x1000;0xaa;000102030405060708090A0B0C0D0E0F;1.000000;1.000000;0.003906;0.003906\n" ~
318         "0x2000;0x55;000102030405060708090A0B0C0D0E0F;1.000000;1.000000;0.007812;0.007812\n" ~
319         "\n";
320     const auto csvcontents = readText(fn);
321     assert(csvcontents == expectedContents, "Mismatched file contents");
322 }