Rev 33 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
| Rev | Author | Line No. | Line |
|---|---|---|---|
| 33 | pmbaty | 1 | #include "chess.h" |
| 2 | #include "data.h" |
||
| 108 | pmbaty | 3 | /* last modified 01/09/15 */ |
| 33 | pmbaty | 4 | /* |
| 5 | ******************************************************************************* |
||
| 6 | * * |
||
| 7 | * Test() is used to test the program against a suite of test positions to * |
||
| 8 | * measure its performance on a particular machine, or to evaluate its skill * |
||
| 9 | * after modifying it in some way. * |
||
| 10 | * * |
||
| 11 | * The test is initiated by using the "test <filename>" command to read in * |
||
| 12 | * the suite of problems from file <filename>. The format of this file is * |
||
| 13 | * as follows: * |
||
| 14 | * * |
||
| 15 | * Setboard <forsythe-string>: This sets the board position using the usual * |
||
| 16 | * forsythe notation (see module SetBoard() in setc for a full ex- * |
||
| 17 | * planation of the syntax). * |
||
| 18 | * * |
||
| 19 | * Solution <move1> <move2> ... <moven>: this provides a solution move (or * |
||
| 20 | * set of solution moves if more than one is correct). If the search finds * |
||
| 21 | * one of these moves, then the prblem is counted as correct, otherwise it * |
||
| 22 | * is counted wrong. * |
||
| 23 | * * |
||
| 24 | * After reading these two lines, the program then searches to whatever time * |
||
| 25 | * or depth limit has been set, when it reaches the end-of-file condition or * |
||
| 26 | * when it reads a record containing the string "end" it then displays the * |
||
| 27 | * number correct and the number missed. * |
||
| 28 | * * |
||
| 29 | * There are two test modules here. Test() handles the specific Crafty test * |
||
| 30 | * data format (dates back to Cray Blitz days) while TestEPD() handles the * |
||
| 31 | * EPD-style test positions which is more concise. Other than the parsing * |
||
| 32 | * differences, these are identical modules. * |
||
| 33 | * * |
||
| 34 | ******************************************************************************* |
||
| 35 | */ |
||
| 108 | pmbaty | 36 | void Test(char *filename, FILE * unsolved, int screen, int margin) { |
| 37 | TREE *const tree = block[0]; |
||
| 33 | pmbaty | 38 | FILE *test_input; |
| 39 | uint64_t nodes = 0; |
||
| 108 | pmbaty | 40 | int i, move, right = 0, wrong = 0, correct, time = 0, len, nfailed = 0; |
| 33 | pmbaty | 41 | float avg_depth = 0.0; |
| 108 | pmbaty | 42 | char failed[8][4096], *eof, *delim; |
| 33 | pmbaty | 43 | |
| 44 | /* |
||
| 45 | ************************************************************ |
||
| 46 | * * |
||
| 47 | * Read in the position and then the solutions. After * |
||
| 48 | * executing a search to find the best move (according to * |
||
| 49 | * the program, anyway) compare it against the list of * |
||
| 50 | * solutions and count it right or wrong. * |
||
| 51 | * * |
||
| 52 | ************************************************************ |
||
| 53 | */ |
||
| 108 | pmbaty | 54 | if (!(test_input = fopen(filename, "r"))) { |
| 33 | pmbaty | 55 | printf("file %s does not exist.\n", filename); |
| 56 | return; |
||
| 57 | } |
||
| 58 | Print(4095, "\n"); |
||
| 59 | eof = fgets(buffer, 4096, test_input); |
||
| 108 | pmbaty | 60 | if (!strstr(buffer, "title")) { |
| 33 | pmbaty | 61 | fclose(test_input); |
| 108 | pmbaty | 62 | TestEPD(filename, unsolved, screen, margin); |
| 33 | pmbaty | 63 | return; |
| 64 | } |
||
| 65 | if (book_file) { |
||
| 66 | fclose(book_file); |
||
| 67 | book_file = 0; |
||
| 68 | } |
||
| 69 | if (books_file) { |
||
| 70 | fclose(books_file); |
||
| 71 | books_file = 0; |
||
| 72 | } |
||
| 108 | pmbaty | 73 | fclose(test_input); |
| 74 | test_input = fopen(filename, "r"); |
||
| 33 | pmbaty | 75 | while (1) { |
| 108 | pmbaty | 76 | eof = fgets(buffer, 4096, test_input); |
| 77 | strcpy(failed[nfailed++], buffer); |
||
| 33 | pmbaty | 78 | if (eof) { |
| 79 | delim = strchr(buffer, '\n'); |
||
| 80 | if (delim) |
||
| 81 | *delim = 0; |
||
| 82 | delim = strchr(buffer, '\r'); |
||
| 83 | if (delim) |
||
| 84 | *delim = ' '; |
||
| 85 | } else |
||
| 86 | break; |
||
| 108 | pmbaty | 87 | nargs = ReadParse(buffer, args, " \t;"); |
| 33 | pmbaty | 88 | if (!strcmp(args[0], "end")) |
| 89 | break; |
||
| 90 | else if (!strcmp(args[0], "title")) { |
||
| 91 | Print(4095, |
||
| 92 | "==============================================" |
||
| 93 | "========================\n"); |
||
| 94 | Print(4095, "! "); |
||
| 95 | len = 0; |
||
| 96 | for (i = 1; i < nargs; i++) { |
||
| 97 | Print(4095, "%s ", args[i]); |
||
| 98 | len += strlen(args[i]) + 1; |
||
| 99 | if (len > 65) |
||
| 100 | break; |
||
| 101 | } |
||
| 102 | for (i = len; i < 67; i++) |
||
| 103 | printf(" "); |
||
| 104 | Print(4095, "!\n"); |
||
| 105 | Print(4095, |
||
| 106 | "==============================================" |
||
| 107 | "========================\n"); |
||
| 108 | } else if (strcmp(args[0], "solution")) { |
||
| 109 | Option(tree); |
||
| 110 | } else { |
||
| 111 | number_of_solutions = 0; |
||
| 112 | solution_type = 0; |
||
| 113 | Print(4095, "solution "); |
||
| 114 | for (i = 1; i < nargs; i++) { |
||
| 115 | if (args[i][strlen(args[i]) - 1] == '?') { |
||
| 116 | solution_type = 1; |
||
| 117 | args[i][strlen(args[i]) - 1] = '\0'; |
||
| 118 | } else if (*(args + i)[strlen(args[i]) - 1] == '!') { |
||
| 119 | solution_type = 0; |
||
| 120 | args[i][strlen(args[i]) - 1] = '\0'; |
||
| 121 | } |
||
| 108 | pmbaty | 122 | move = InputMove(tree, 0, game_wtm, 0, 0, args[i]); |
| 33 | pmbaty | 123 | if (move) { |
| 124 | solutions[number_of_solutions] = move; |
||
| 125 | Print(4095, "%d. %s", (number_of_solutions++) + 1, OutputMove(tree, |
||
| 108 | pmbaty | 126 | 0, game_wtm, move)); |
| 33 | pmbaty | 127 | if (solution_type == 1) |
| 128 | Print(4095, "? "); |
||
| 129 | else |
||
| 130 | Print(4095, " "); |
||
| 131 | } else |
||
| 132 | DisplayChessBoard(stdout, tree->position); |
||
| 133 | } |
||
| 134 | Print(4095, "\n"); |
||
| 108 | pmbaty | 135 | InitializeHashTables(0); |
| 33 | pmbaty | 136 | last_pv.pathd = 0; |
| 137 | thinking = 1; |
||
| 138 | tree->status[1] = tree->status[0]; |
||
| 108 | pmbaty | 139 | Iterate(game_wtm, think, 0); |
| 33 | pmbaty | 140 | thinking = 0; |
| 141 | nodes += tree->nodes_searched; |
||
| 108 | pmbaty | 142 | avg_depth += (float) iteration; |
| 33 | pmbaty | 143 | time += (end_time - start_time); |
| 144 | correct = solution_type; |
||
| 145 | for (i = 0; i < number_of_solutions; i++) { |
||
| 146 | if (!solution_type) { |
||
| 108 | pmbaty | 147 | if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff)) |
| 33 | pmbaty | 148 | correct = 1; |
| 108 | pmbaty | 149 | } else if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff)) |
| 33 | pmbaty | 150 | correct = 0; |
| 151 | } |
||
| 152 | if (correct) { |
||
| 153 | right++; |
||
| 154 | Print(4095, "----------------------> solution correct (%d/%d).\n", |
||
| 155 | right, right + wrong); |
||
| 156 | } else { |
||
| 157 | wrong++; |
||
| 158 | Print(4095, "----------------------> solution incorrect (%d/%d).\n", |
||
| 159 | right, right + wrong); |
||
| 108 | pmbaty | 160 | if (unsolved) |
| 161 | for (i = 0; i < nfailed; i++) |
||
| 162 | fputs(failed[i], unsolved); |
||
| 33 | pmbaty | 163 | } |
| 108 | pmbaty | 164 | nfailed = 0; |
| 33 | pmbaty | 165 | } |
| 166 | } |
||
| 167 | /* |
||
| 168 | ************************************************************ |
||
| 169 | * * |
||
| 170 | * Now print the results. * |
||
| 171 | * * |
||
| 172 | ************************************************************ |
||
| 173 | */ |
||
| 174 | if (right + wrong) { |
||
| 175 | Print(4095, "\n\n\n"); |
||
| 176 | Print(4095, "test results summary:\n\n"); |
||
| 177 | Print(4095, "total positions searched..........%12d\n", right + wrong); |
||
| 178 | Print(4095, "number right......................%12d\n", right); |
||
| 179 | Print(4095, "number wrong......................%12d\n", wrong); |
||
| 180 | Print(4095, "percentage right..................%12d\n", |
||
| 181 | right * 100 / (right + wrong)); |
||
| 182 | Print(4095, "percentage wrong..................%12d\n", |
||
| 183 | wrong * 100 / (right + wrong)); |
||
| 184 | Print(4095, "total nodes searched..............%12" PRIu64 "\n", nodes); |
||
| 185 | Print(4095, "average search depth..............%12.1f\n", |
||
| 186 | avg_depth / (right + wrong)); |
||
| 187 | Print(4095, "nodes per second..................%12" PRIu64 "\n", |
||
| 188 | nodes * 100 / Max(time, 1)); |
||
| 189 | Print(4095, "total time........................%12s\n", |
||
| 190 | DisplayTime(time)); |
||
| 191 | } |
||
| 192 | input_stream = stdin; |
||
| 193 | early_exit = 99; |
||
| 194 | } |
||
| 195 | |||
| 108 | pmbaty | 196 | /* last modified 06/26/15 */ |
| 33 | pmbaty | 197 | /* |
| 198 | ******************************************************************************* |
||
| 199 | * * |
||
| 200 | * TestEPD() is used to test the program against a suite of test positions * |
||
| 201 | * to measure its performance on a particular machine, or to evaluate its * |
||
| 202 | * skill after modifying it in some way. * |
||
| 203 | * * |
||
| 204 | * The test is initiated by using the "test <filename>" command to read in * |
||
| 205 | * the suite of problems from file <filename>. The format of this file is * |
||
| 206 | * as follows: * |
||
| 207 | * * |
||
| 208 | * <forsythe-string> am/bm move1 move2 etc; title "xxx" * |
||
| 209 | * * |
||
| 210 | * Am means "avoid move" and bm means "best move". Each test position may * |
||
| 211 | * have multiple moves to avoid or that are best, but both am and bm may not * |
||
| 212 | * appear on one position. * |
||
| 213 | * * |
||
| 214 | * The title is just a comment that is given in the program output to make * |
||
| 215 | * it easier to match output to specific positions. * |
||
| 216 | * * |
||
| 108 | pmbaty | 217 | * One new addition is the ability to take a set of EPD records and run a * |
| 218 | * search on each one. If the final evaluation is within some window, then * |
||
| 219 | * the input record is written out to a second file. This is used to screen * |
||
| 220 | * cluster-testing starting positions to weed out those that are so badly * |
||
| 221 | * unbalanced that one side always wins. * |
||
| 222 | * * |
||
| 33 | pmbaty | 223 | ******************************************************************************* |
| 224 | */ |
||
| 108 | pmbaty | 225 | void TestEPD(char *filename, FILE * unsolved, int screen, int margin) { |
| 226 | TREE *const tree = block[0]; |
||
| 227 | FILE *test_input, *test_output = 0; |
||
| 33 | pmbaty | 228 | uint64_t nodes = 0; |
| 108 | pmbaty | 229 | int i, move, right = 0, wrong = 0, correct, time = 0, len, culled = 0, r = |
| 230 | 0; |
||
| 33 | pmbaty | 231 | float avg_depth = 0.0; |
| 108 | pmbaty | 232 | char *eof, *mvs, *title, tbuffer[512], failed[4096]; |
| 33 | pmbaty | 233 | |
| 234 | /* |
||
| 235 | ************************************************************ |
||
| 236 | * * |
||
| 237 | * Read in the position and then the solutions. After * |
||
| 238 | * executing a search to find the best move (according to * |
||
| 239 | * the program, anyway) compare it against the list of * |
||
| 240 | * solutions and count it right or wrong. * |
||
| 241 | * * |
||
| 242 | ************************************************************ |
||
| 243 | */ |
||
| 108 | pmbaty | 244 | if (!(test_input = fopen(filename, "r"))) { |
| 33 | pmbaty | 245 | printf("file %s does not exist.\n", filename); |
| 246 | return; |
||
| 247 | } |
||
| 108 | pmbaty | 248 | if (screen) { |
| 249 | char outfile[256]; |
||
| 250 | |||
| 251 | strcpy(outfile, filename); |
||
| 252 | strcat(outfile, ".screened"); |
||
| 253 | if (!(test_output = fopen(outfile, "w"))) { |
||
| 254 | printf("file %s cannot be opened for write.\n", filename); |
||
| 255 | return; |
||
| 256 | } |
||
| 257 | } |
||
| 33 | pmbaty | 258 | if (book_file) { |
| 259 | fclose(book_file); |
||
| 260 | book_file = 0; |
||
| 261 | } |
||
| 262 | if (books_file) { |
||
| 263 | fclose(books_file); |
||
| 264 | books_file = 0; |
||
| 265 | } |
||
| 266 | while (1) { |
||
| 267 | eof = fgets(buffer, 4096, test_input); |
||
| 108 | pmbaty | 268 | strcpy(failed, buffer); |
| 269 | Print(4095, "%s\n", buffer); |
||
| 270 | strcpy(tbuffer, buffer); |
||
| 33 | pmbaty | 271 | if (eof) { |
| 272 | char *delim; |
||
| 273 | |||
| 274 | delim = strchr(buffer, '\n'); |
||
| 275 | if (delim) |
||
| 276 | *delim = 0; |
||
| 277 | delim = strchr(buffer, '\r'); |
||
| 278 | if (delim) |
||
| 279 | *delim = ' '; |
||
| 280 | } else |
||
| 281 | break; |
||
| 108 | pmbaty | 282 | r++; |
| 283 | mvs = strstr(buffer, " sd "); |
||
| 284 | if (mvs) { |
||
| 285 | search_depth = atoi(mvs + 3); |
||
| 286 | *(mvs - 1) = 0; |
||
| 287 | Print(4095, "search depth %d\n", search_depth); |
||
| 288 | } |
||
| 33 | pmbaty | 289 | mvs = strstr(buffer, " bm "); |
| 290 | if (!mvs) |
||
| 291 | mvs = strstr(buffer, " am "); |
||
| 108 | pmbaty | 292 | if (!mvs && !screen) |
| 33 | pmbaty | 293 | Print(4095, "Warning. am/bm field missing, input string follows\n%s\n", |
| 294 | buffer); |
||
| 295 | if (mvs) |
||
| 296 | mvs++; |
||
| 297 | title = strstr(buffer, "id"); |
||
| 298 | if (mvs) |
||
| 299 | *(mvs - 1) = 0; |
||
| 300 | if (title) |
||
| 301 | *(title - 1) = 0; |
||
| 302 | if (title) { |
||
| 303 | title = strchr(title, '\"') + 1; |
||
| 304 | if (title) { |
||
| 305 | if (strchr(title, '\"')) { |
||
| 306 | *strchr(title, '\"') = 0; |
||
| 307 | } |
||
| 308 | } |
||
| 309 | Print(4095, |
||
| 310 | "==============================================" |
||
| 311 | "========================\n"); |
||
| 312 | Print(4095, "! "); |
||
| 313 | Print(4095, "%s ", title); |
||
| 314 | len = 66 - strlen(title); |
||
| 315 | for (i = 0; i < len; i++) |
||
| 316 | printf(" "); |
||
| 317 | Print(4095, "!\n"); |
||
| 318 | Print(4095, |
||
| 319 | "==============================================" |
||
| 320 | "========================\n"); |
||
| 321 | } |
||
| 322 | Option(tree); |
||
| 323 | if (mvs) { |
||
| 108 | pmbaty | 324 | nargs = ReadParse(mvs, args, " \t;"); |
| 33 | pmbaty | 325 | number_of_solutions = 0; |
| 326 | solution_type = 0; |
||
| 327 | if (!strcmp(args[0], "am")) |
||
| 328 | solution_type = 1; |
||
| 329 | Print(4095, "solution "); |
||
| 330 | for (i = 1; i < nargs; i++) { |
||
| 331 | if (!strcmp(args[i], "c0")) |
||
| 332 | break; |
||
| 108 | pmbaty | 333 | move = InputMove(tree, 0, game_wtm, 0, 0, args[i]); |
| 33 | pmbaty | 334 | if (move) { |
| 335 | solutions[number_of_solutions] = move; |
||
| 336 | Print(4095, "%d. %s", (number_of_solutions++) + 1, OutputMove(tree, |
||
| 108 | pmbaty | 337 | 0, game_wtm, move)); |
| 33 | pmbaty | 338 | if (solution_type == 1) |
| 339 | Print(4095, "? "); |
||
| 340 | else |
||
| 341 | Print(4095, " "); |
||
| 342 | } else |
||
| 343 | DisplayChessBoard(stdout, tree->position); |
||
| 344 | } |
||
| 345 | } |
||
| 346 | Print(4095, "\n"); |
||
| 108 | pmbaty | 347 | InitializeHashTables(0); |
| 33 | pmbaty | 348 | last_pv.pathd = 0; |
| 349 | thinking = 1; |
||
| 350 | tree->status[1] = tree->status[0]; |
||
| 108 | pmbaty | 351 | Iterate(game_wtm, think, 0); |
| 352 | if (screen) { |
||
| 353 | if (Abs(last_root_value) < margin) |
||
| 354 | fwrite(tbuffer, 1, strlen(tbuffer), test_output); |
||
| 355 | else |
||
| 356 | culled++; |
||
| 357 | printf("record #%d, culled %d, score=%s \r", r, culled, |
||
| 358 | DisplayEvaluation(last_root_value, game_wtm)); |
||
| 359 | fflush(stdout); |
||
| 360 | } |
||
| 33 | pmbaty | 361 | thinking = 0; |
| 362 | nodes += tree->nodes_searched; |
||
| 108 | pmbaty | 363 | avg_depth += (float) iteration; |
| 33 | pmbaty | 364 | time += (end_time - start_time); |
| 108 | pmbaty | 365 | if (!screen) { |
| 366 | correct = solution_type; |
||
| 367 | for (i = 0; i < number_of_solutions; i++) { |
||
| 368 | if (!solution_type) { |
||
| 369 | if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff)) |
||
| 370 | correct = 1; |
||
| 371 | } else if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff)) |
||
| 372 | correct = 0; |
||
| 373 | } |
||
| 374 | if (correct) { |
||
| 375 | right++; |
||
| 376 | Print(4095, "----------------------> solution correct (%d/%d).\n", |
||
| 377 | right, right + wrong); |
||
| 378 | } else { |
||
| 379 | wrong++; |
||
| 380 | Print(4095, "----------------------> solution incorrect (%d/%d).\n", |
||
| 381 | right, right + wrong); |
||
| 382 | if (unsolved) |
||
| 383 | fputs(failed, unsolved); |
||
| 384 | } |
||
| 33 | pmbaty | 385 | } |
| 386 | } |
||
| 387 | /* |
||
| 388 | ************************************************************ |
||
| 389 | * * |
||
| 390 | * Now print the results. * |
||
| 391 | * * |
||
| 392 | ************************************************************ |
||
| 393 | */ |
||
| 108 | pmbaty | 394 | if (r) { |
| 33 | pmbaty | 395 | Print(4095, "\n\n\n"); |
| 396 | Print(4095, "test results summary:\n\n"); |
||
| 108 | pmbaty | 397 | Print(4095, "total positions searched..........%12d\n", r); |
| 398 | if (!screen) { |
||
| 399 | Print(4095, "number right......................%12d\n", right); |
||
| 400 | Print(4095, "number wrong......................%12d\n", wrong); |
||
| 401 | Print(4095, "percentage right..................%12d\n", |
||
| 402 | right * 100 / (right + wrong)); |
||
| 403 | Print(4095, "percentage wrong..................%12d\n", |
||
| 404 | wrong * 100 / (right + wrong)); |
||
| 405 | } else |
||
| 406 | Print(4095, "records excluded..................%12d\n", culled); |
||
| 407 | |||
| 33 | pmbaty | 408 | Print(4095, "total nodes searched..............%12" PRIu64 "\n", nodes); |
| 108 | pmbaty | 409 | Print(4095, "average search depth..............%12.1f\n", avg_depth / r); |
| 33 | pmbaty | 410 | Print(4095, "nodes per second..................%12" PRIu64 "\n", |
| 411 | nodes * 100 / Max(1, time)); |
||
| 412 | Print(4095, "total time........................%12s\n", |
||
| 413 | DisplayTime(time)); |
||
| 414 | } |
||
| 415 | input_stream = stdin; |
||
| 416 | early_exit = 99; |
||
| 417 | } |