Subversion Repositories Games.Chess Giants

Rev

Rev 108 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. #include "chess.h"
  2. #include "data.h"
  3. /* last modified 01/09/15 */
  4. /*
  5.  *******************************************************************************
  6.  *                                                                             *
  7.  *   Test() is used to test the program against a suite of test positions to   *
  8.  *   measure its performance on a particular machine, or to evaluate its skill *
  9.  *   after modifying it in some way.                                           *
  10.  *                                                                             *
  11.  *   The test is initiated by using the "test <filename>" command to read in   *
  12.  *   the suite of problems from file <filename>.  The format of this file is   *
  13.  *   as follows:                                                               *
  14.  *                                                                             *
  15.  *   Setboard <forsythe-string>:  This sets the board position using the usual *
  16.  *   forsythe notation (see module SetBoard() in setc for a full ex-           *
  17.  *   planation of the syntax).                                                 *
  18.  *                                                                             *
  19.  *   Solution <move1> <move2> ... <moven>:  this provides a solution move (or  *
  20.  *   set of solution moves if more than one is correct).  If the search finds  *
  21.  *   one of these moves, then the prblem is counted as correct, otherwise it   *
  22.  *   is counted wrong.                                                         *
  23.  *                                                                             *
  24.  *   After reading these two lines, the program then searches to whatever time *
  25.  *   or depth limit has been set, when it reaches the end-of-file condition or *
  26.  *   when it reads a record containing the string "end" it then displays the   *
  27.  *   number correct and the number missed.                                     *
  28.  *                                                                             *
  29.  *   There are two test modules here.  Test() handles the specific Crafty test *
  30.  *   data format (dates back to Cray Blitz days) while TestEPD() handles the   *
  31.  *   EPD-style test positions which is more concise.  Other than the parsing   *
  32.  *   differences, these are identical modules.                                 *
  33.  *                                                                             *
  34.  *******************************************************************************
  35.  */
  36. void Test(char *filename, FILE * unsolved, int screen, int margin) {
  37.   TREE *const tree = block[0];
  38.   FILE *test_input;
  39.   uint64_t nodes = 0;
  40.   int i, move, right = 0, wrong = 0, correct, time = 0, len, nfailed = 0;
  41.   float avg_depth = 0.0;
  42.   char failed[8][4096], *eof, *delim;
  43.  
  44. /*
  45.  ************************************************************
  46.  *                                                          *
  47.  *  Read in the position and then the solutions.  After     *
  48.  *  executing a search to find the best move (according to  *
  49.  *  the program, anyway) compare it against the list of     *
  50.  *  solutions and count it right or wrong.                  *
  51.  *                                                          *
  52.  ************************************************************
  53.  */
  54.   if (!(test_input = fopen(filename, "r"))) {
  55.     printf("file %s does not exist.\n", filename);
  56.     return;
  57.   }
  58.   Print(4095, "\n");
  59.   eof = fgets(buffer, 4096, test_input);
  60.   if (!strstr(buffer, "title")) {
  61.     fclose(test_input);
  62.     TestEPD(filename, unsolved, screen, margin);
  63.     return;
  64.   }
  65.   if (book_file) {
  66.     fclose(book_file);
  67.     book_file = 0;
  68.   }
  69.   if (books_file) {
  70.     fclose(books_file);
  71.     books_file = 0;
  72.   }
  73.   fclose(test_input);
  74.   test_input = fopen(filename, "r");
  75.   while (FOREVER) {
  76.     eof = fgets(buffer, 4096, test_input);
  77.     strcpy(failed[nfailed++], buffer);
  78.     if (eof) {
  79.       delim = strchr(buffer, '\n');
  80.       if (delim)
  81.         *delim = 0;
  82.       delim = strchr(buffer, '\r');
  83.       if (delim)
  84.         *delim = ' ';
  85.     } else
  86.       break;
  87.     nargs = ReadParse(buffer, args, " \t;");
  88.     if (!strcmp(args[0], "end"))
  89.       break;
  90.     else if (!strcmp(args[0], "title")) {
  91.       Print(4095,
  92.           "=============================================="
  93.           "========================\n");
  94.       Print(4095, "! ");
  95.       len = 0;
  96.       for (i = 1; i < nargs; i++) {
  97.         Print(4095, "%s ", args[i]);
  98.         len += strlen(args[i]) + 1;
  99.         if (len > 65)
  100.           break;
  101.       }
  102.       for (i = len; i < 67; i++)
  103.         printf(" ");
  104.       Print(4095, "!\n");
  105.       Print(4095,
  106.           "=============================================="
  107.           "========================\n");
  108.     } else if (strcmp(args[0], "solution")) {
  109.       Option(tree);
  110.     } else {
  111.       number_of_solutions = 0;
  112.       solution_type = 0;
  113.       Print(4095, "solution ");
  114.       for (i = 1; i < nargs; i++) {
  115.         if (args[i][strlen(args[i]) - 1] == '?') {
  116.           solution_type = 1;
  117.           args[i][strlen(args[i]) - 1] = '\0';
  118.         } else if (*(args + i)[strlen(args[i]) - 1] == '!') {
  119.           solution_type = 0;
  120.           args[i][strlen(args[i]) - 1] = '\0';
  121.         }
  122.         move = InputMove(tree, 0, game_wtm, 0, 0, args[i]);
  123.         if (move) {
  124.           solutions[number_of_solutions] = move;
  125.           Print(4095, "%d. %s", (number_of_solutions++) + 1, OutputMove(tree,
  126.                   0, game_wtm, move));
  127.           if (solution_type == 1)
  128.             Print(4095, "? ");
  129.           else
  130.             Print(4095, "  ");
  131.         } else
  132.           DisplayChessBoard(stdout, tree->position);
  133.       }
  134.       Print(4095, "\n");
  135.       InitializeHashTables(0);
  136.       last_pv.pathd = 0;
  137.       thinking = 1;
  138.       tree->status[1] = tree->status[0];
  139.       Iterate(game_wtm, think, 0);
  140.       thinking = 0;
  141.       nodes += tree->nodes_searched;
  142.       avg_depth += (float) iteration;
  143.       time += (end_time - start_time);
  144.       correct = solution_type;
  145.       for (i = 0; i < number_of_solutions; i++) {
  146.         if (!solution_type) {
  147.           if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
  148.             correct = 1;
  149.         } else if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
  150.           correct = 0;
  151.       }
  152.       if (correct) {
  153.         right++;
  154.         Print(4095, "----------------------> solution correct (%d/%d).\n",
  155.             right, right + wrong);
  156.       } else {
  157.         wrong++;
  158.         Print(4095, "----------------------> solution incorrect (%d/%d).\n",
  159.             right, right + wrong);
  160.         if (unsolved)
  161.           for (i = 0; i < nfailed; i++)
  162.             fputs(failed[i], unsolved);
  163.       }
  164.       nfailed = 0;
  165.     }
  166.   }
  167. /*
  168.  ************************************************************
  169.  *                                                          *
  170.  *  Now print the results.                                  *
  171.  *                                                          *
  172.  ************************************************************
  173.  */
  174.   if (right + wrong) {
  175.     Print(4095, "\n\n\n");
  176.     Print(4095, "test results summary:\n\n");
  177.     Print(4095, "total positions searched..........%12d\n", right + wrong);
  178.     Print(4095, "number right......................%12d\n", right);
  179.     Print(4095, "number wrong......................%12d\n", wrong);
  180.     Print(4095, "percentage right..................%12d\n",
  181.         right * 100 / (right + wrong));
  182.     Print(4095, "percentage wrong..................%12d\n",
  183.         wrong * 100 / (right + wrong));
  184.     Print(4095, "total nodes searched..............%12" PRIu64 "\n", nodes);
  185.     Print(4095, "average search depth..............%12.1f\n",
  186.         avg_depth / (right + wrong));
  187.     Print(4095, "nodes per second..................%12" PRIu64 "\n",
  188.         nodes * 100 / Max(time, 1));
  189.     Print(4095, "total time........................%12s\n",
  190.         DisplayTime(time));
  191.   }
  192.   input_stream = stdin;
  193.   early_exit = 99;
  194.   fclose(test_input);
  195. }
  196.  
  197. /* last modified 06/26/15 */
  198. /*
  199.  *******************************************************************************
  200.  *                                                                             *
  201.  *   TestEPD() is used to test the program against a suite of test positions   *
  202.  *   to measure its performance on a particular machine, or to evaluate its    *
  203.  *   skill after modifying it in some way.                                     *
  204.  *                                                                             *
  205.  *   The test is initiated by using the "test <filename>" command to read in   *
  206.  *   the suite of problems from file <filename>.  The format of this file is   *
  207.  *   as follows:                                                               *
  208.  *                                                                             *
  209.  *   <forsythe-string>  am/bm move1 move2 etc; title "xxx"                     *
  210.  *                                                                             *
  211.  *   Am means "avoid move" and bm means "best move".  Each test position may   *
  212.  *   have multiple moves to avoid or that are best, but both am and bm may not *
  213.  *   appear on one position.                                                   *
  214.  *                                                                             *
  215.  *   The title is just a comment that is given in the program output to make   *
  216.  *   it easier to match output to specific positions.                          *
  217.  *                                                                             *
  218.  *   One new addition is the ability to take a set of EPD records and run a    *
  219.  *   search on each one.  If the final evaluation is within some window, then  *
  220.  *   the input record is written out to a second file.  This is used to screen *
  221.  *   cluster-testing starting positions to weed out those that are so badly    *
  222.  *   unbalanced that one side always wins.                                     *
  223.  *                                                                             *
  224.  *******************************************************************************
  225.  */
  226. void TestEPD(char *filename, FILE * unsolved, int screen, int margin) {
  227.   TREE *const tree = block[0];
  228.   FILE *test_input, *test_output = 0;
  229.   uint64_t nodes = 0;
  230.   int i, move, right = 0, wrong = 0, correct, time = 0, len, culled = 0, r =
  231.       0;
  232.   float avg_depth = 0.0;
  233.   char *eof, *mvs, *title, tbuffer[512], failed[4096];
  234.  
  235. /*
  236.  ************************************************************
  237.  *                                                          *
  238.  *  Read in the position and then the solutions.  After     *
  239.  *  executing a search to find the best move (according to  *
  240.  *  the program, anyway) compare it against the list of     *
  241.  *  solutions and count it right or wrong.                  *
  242.  *                                                          *
  243.  ************************************************************
  244.  */
  245.   if (!(test_input = fopen(filename, "r"))) {
  246.     printf("file %s does not exist.\n", filename);
  247.     return;
  248.   }
  249.   if (screen) {
  250.     char outfile[256];
  251.  
  252.     strcpy(outfile, filename);
  253.     strcat(outfile, ".screened");
  254.     if (!(test_output = fopen(outfile, "w"))) {
  255.       printf("file %s cannot be opened for write.\n", filename);
  256.       return;
  257.     }
  258.   }
  259.   if (book_file) {
  260.     fclose(book_file);
  261.     book_file = 0;
  262.   }
  263.   if (books_file) {
  264.     fclose(books_file);
  265.     books_file = 0;
  266.   }
  267.   while (FOREVER) {
  268.     eof = fgets(buffer, 4096, test_input);
  269.     strcpy(failed, buffer);
  270.     Print(4095, "%s\n", buffer);
  271.     strcpy(tbuffer, buffer);
  272.     if (eof) {
  273.       char *delim;
  274.  
  275.       delim = strchr(buffer, '\n');
  276.       if (delim)
  277.         *delim = 0;
  278.       delim = strchr(buffer, '\r');
  279.       if (delim)
  280.         *delim = ' ';
  281.     } else
  282.       break;
  283.     r++;
  284.     mvs = strstr(buffer, " sd ");
  285.     if (mvs) {
  286.       search_depth = atoi(mvs + 3);
  287.       *(mvs - 1) = 0;
  288.       Print(4095, "search depth %d\n", search_depth);
  289.     }
  290.     mvs = strstr(buffer, " bm ");
  291.     if (!mvs)
  292.       mvs = strstr(buffer, " am ");
  293.     if (!mvs && !screen)
  294.       Print(4095, "Warning. am/bm field missing, input string follows\n%s\n",
  295.           buffer);
  296.     if (mvs)
  297.       mvs++;
  298.     title = strstr(buffer, "id");
  299.     if (mvs)
  300.       *(mvs - 1) = 0;
  301.     if (title)
  302.       *(title - 1) = 0;
  303.     if (title) {
  304.       title = strchr(title, '\"') + 1;
  305.       if (title) {
  306.         if (strchr(title, '\"')) {
  307.           *strchr(title, '\"') = 0;
  308.         }
  309.       }
  310.       Print(4095,
  311.           "=============================================="
  312.           "========================\n");
  313.       Print(4095, "! ");
  314.       Print(4095, "%s ", title);
  315.       len = 66 - strlen(title);
  316.       for (i = 0; i < len; i++)
  317.         printf(" ");
  318.       Print(4095, "!\n");
  319.       Print(4095,
  320.           "=============================================="
  321.           "========================\n");
  322.     }
  323.     Option(tree);
  324.     if (mvs) {
  325.       nargs = ReadParse(mvs, args, " \t;");
  326.       number_of_solutions = 0;
  327.       solution_type = 0;
  328.       if (!strcmp(args[0], "am"))
  329.         solution_type = 1;
  330.       Print(4095, "solution ");
  331.       for (i = 1; i < nargs; i++) {
  332.         if (!strcmp(args[i], "c0"))
  333.           break;
  334.         move = InputMove(tree, 0, game_wtm, 0, 0, args[i]);
  335.         if (move) {
  336.           solutions[number_of_solutions] = move;
  337.           Print(4095, "%d. %s", (number_of_solutions++) + 1, OutputMove(tree,
  338.                   0, game_wtm, move));
  339.           if (solution_type == 1)
  340.             Print(4095, "? ");
  341.           else
  342.             Print(4095, "  ");
  343.         } else
  344.           DisplayChessBoard(stdout, tree->position);
  345.       }
  346.     }
  347.     Print(4095, "\n");
  348.     InitializeHashTables(0);
  349.     last_pv.pathd = 0;
  350.     thinking = 1;
  351.     tree->status[1] = tree->status[0];
  352.     Iterate(game_wtm, think, 0);
  353.     if (screen) {
  354.       if (Abs(last_root_value) < margin)
  355.         fwrite(tbuffer, 1, strlen(tbuffer), test_output);
  356.       else
  357.         culled++;
  358.       printf("record #%d,  culled %d, score=%s          \r", r, culled,
  359.           DisplayEvaluation(last_root_value, game_wtm));
  360.       fflush(stdout);
  361.     }
  362.     thinking = 0;
  363.     nodes += tree->nodes_searched;
  364.     avg_depth += (float) iteration;
  365.     time += (end_time - start_time);
  366.     if (!screen) {
  367.       correct = solution_type;
  368.       for (i = 0; i < number_of_solutions; i++) {
  369.         if (!solution_type) {
  370.           if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
  371.             correct = 1;
  372.         } else if (solutions[i] == (tree->pv[1].path[1] & 0x001fffff))
  373.           correct = 0;
  374.       }
  375.       if (correct) {
  376.         right++;
  377.         Print(4095, "----------------------> solution correct (%d/%d).\n",
  378.             right, right + wrong);
  379.       } else {
  380.         wrong++;
  381.         Print(4095, "----------------------> solution incorrect (%d/%d).\n",
  382.             right, right + wrong);
  383.         if (unsolved)
  384.           fputs(failed, unsolved);
  385.       }
  386.     }
  387.   }
  388. /*
  389.  ************************************************************
  390.  *                                                          *
  391.  *  Now print the results.                                  *
  392.  *                                                          *
  393.  ************************************************************
  394.  */
  395.   if (r) {
  396.     Print(4095, "\n\n\n");
  397.     Print(4095, "test results summary:\n\n");
  398.     Print(4095, "total positions searched..........%12d\n", r);
  399.     if (!screen) {
  400.       Print(4095, "number right......................%12d\n", right);
  401.       Print(4095, "number wrong......................%12d\n", wrong);
  402.       Print(4095, "percentage right..................%12d\n",
  403.           right * 100 / (right + wrong));
  404.       Print(4095, "percentage wrong..................%12d\n",
  405.           wrong * 100 / (right + wrong));
  406.     } else
  407.       Print(4095, "records excluded..................%12d\n", culled);
  408.  
  409.     Print(4095, "total nodes searched..............%12" PRIu64 "\n", nodes);
  410.     Print(4095, "average search depth..............%12.1f\n", avg_depth / r);
  411.     Print(4095, "nodes per second..................%12" PRIu64 "\n",
  412.         nodes * 100 / Max(1, time));
  413.     Print(4095, "total time........................%12s\n",
  414.         DisplayTime(time));
  415.   }
  416.   input_stream = stdin;
  417.   early_exit = 99;
  418.   fclose(test_input);
  419. }
  420.