MySQL 5.6.14 Source Code Document
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
perf.cpp
1 /*
2  Copyright (C) 2003-2006 MySQL AB
3  All rights reserved. Use is subject to license terms.
4 
5  This program is free software; you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation; version 2 of the License.
8 
9  This program is distributed in the hope that it will be useful,
10  but WITHOUT ANY WARRANTY; without even the implied warranty of
11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  GNU General Public License for more details.
13 
14  You should have received a copy of the GNU General Public License
15  along with this program; if not, write to the Free Software
16  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18 
19 
20 #include <ndb_global.h>
21 
22 extern "C" {
23 #include <dba.h>
24 }
25 
26 #include <NdbOut.hpp>
27 #include <NdbSleep.h>
28 #include <NdbTimer.hpp>
29 #include <NDBT_Stats.hpp>
30 #include <NDBT_ReturnCodes.h>
31 #include <NdbMain.h>
32 #include <time.h>
33 
34 #undef min
35 #undef max
36 
37 static const int NP_Insert = 0;
38 static const int NP_Update = 1;
39 static const int NP_WriteUpdate = 2;
40 static const int NP_WriteInsert = 3;
41 static const int NP_Delete = 4;
42 static const int NP_BulkRead = 5;
43 static const int NP_MAX = 5;
44 
45 static const char * Operations[] = {
46  "Insert ",
47  "Update ",
48  "WriteUpd",
49  "WriteIns",
50  "Delete ",
51  "BulkRead"
52 };
53 
57 static int NoOfTransactions = 10000;
58 static int ParallellTransactions = 1000;
59 static int OperationsPerTransaction = 10;
60 static int NoOfColumns = 20;
61 static int BytesPerInsert = 300;
62 static int BytesPerUpdate = 200;
63 static int LoopCount = 10;
64 
68 static char TableName[255];
69 static DBA_ColumnDesc_t * ColumnDescriptions;
70 static DBA_ColumnBinding_t * InsertBindings;
71 static DBA_ColumnBinding_t * UpdateBindings; static int UpdateBindingColumns;
72 static DBA_ColumnBinding_t * DeleteBindings;
73 
74 static char * TestData;
75 static DBA_Binding_t * InsertB;
76 static DBA_Binding_t * UpdateB;
77 static DBA_Binding_t * DeleteB;
78 
82 static void sequence(int loops);
83 
84 inline void * getPtr(int rowNo) { return TestData+rowNo*BytesPerInsert;}
85 inline void setPK(int rowNo, int pk){ * (int *)getPtr(rowNo) = pk; }
86 
87 static void SetupTestData();
88 static void CleanupTestData();
89 
90 static bool CreateTable();
91 static bool CleanTable();
92 static bool CreateBindings();
93 
94 static void usage();
95 
96 static
97 void
98 usage(){
99  int ForceSend, Interval;
100  DBA_GetParameter(0, &Interval);
101  DBA_GetParameter(3, &ForceSend);
102 
103  ndbout << "newtonPerf" << endl
104  << " -n Transactions per loop and operation ("
105  << NoOfTransactions << ")" << endl
106  << " -p parallell transactions (" << ParallellTransactions << ")"
107  << endl
108  << " -o operations per transaction (" << OperationsPerTransaction
109  << ")" << endl
110  << " -a no of columns (" << NoOfColumns << ")" << endl
111  << " -b Table size in bytes (" << BytesPerInsert << ")" << endl
112  << " -u Bytes per update (" << BytesPerUpdate << ")" << endl
113  << " -l Loop count (" << LoopCount << ")" << endl
114  << " -i Interval (" << Interval << "ms)" << endl
115  << " -f Force send algorithm (" << ForceSend << ")" << endl
116  << " -h Help" << endl;
117 
118 }
119 
120 static
121 bool
122 parseArgs(int argc, const char **argv){
123  bool a = false, b = false, u = false;
124 
125  for(int i = 1; i<argc; i++){
126  if(argv[i][0] != '-'){
127  ndbout << "Invalid argument: " << argv[i] << endl;
128  return false;
129  }
130 
131  if(argv[i][1] == 'h')
132  return false;
133 
134  if(i == argc-1){
135  ndbout << "Expecting argument to " << argv[i] << endl;
136  return false;
137  }
138 
139  switch(argv[i][1]){
140  case 'n':
141  NoOfTransactions = atoi(argv[i+1]);
142  break;
143  case 'p':
144  ParallellTransactions = atoi(argv[i+1]);
145  break;
146  case 'o':
147  OperationsPerTransaction = atoi(argv[i+1]);
148  break;
149  case 'a':
150  NoOfColumns = atoi(argv[i+1]);
151  a = true;
152  break;
153  case 'b':
154  BytesPerInsert = atoi(argv[i+1]);
155  b = true;
156  break;
157  case 'u':
158  BytesPerUpdate = atoi(argv[i+1]);
159  u = true;
160  break;
161  case 'l':
162  LoopCount = atoi(argv[i+1]);
163  break;
164  case 'f':
165  {
166  const int val = atoi(argv[i+1]);
167  if(DBA_SetParameter(3, val) != DBA_NO_ERROR){
168  ndbout << "Invalid force send algorithm: "
169  << DBA_GetLatestErrorMsg()
170  << "(" << DBA_GetLatestError() << ")" << endl;
171  return false;
172  }
173  }
174  break;
175  case 'i':
176  {
177  const int val = atoi(argv[i+1]);
178  if(DBA_SetParameter(0, val) != DBA_NO_ERROR){
179  ndbout << "Invalid NBP interval: "
180  << DBA_GetLatestErrorMsg()
181  << "(" << DBA_GetLatestError() << ")" << endl;
182  return false;
183  }
184  }
185  break;
186  default:
187  ndbout << "Invalid option: " << argv[i] << endl;
188  return false;
189  }
190  i++;
191  }
192  if(a && !b) BytesPerInsert = 15 * NoOfColumns;
193  if(!a && b) NoOfColumns = ((BytesPerInsert + 14) / 15)+1;
194 
195  if(!u)
196  BytesPerUpdate = (2 * BytesPerInsert) / 3;
197 
198  bool t = true;
199  if(NoOfColumns < 2) t = false;
200  if(BytesPerInsert < 8) t = false;
201  if(BytesPerUpdate < 8) t = false;
202 
203  if(!t){
204  ndbout << "Invalid arguments combination of -a -b -u not working out"
205  << endl;
206  return false;
207  }
208  return true;
209 }
210 
211 NDB_COMMAND(newton_perf, "newton_perf",
212  "newton_perf", "newton_perf", 65535){
213 
214  if(!parseArgs(argc, argv)){
215  usage();
216  return NDBT_ProgramExit(NDBT_WRONGARGS);
217  }
218 
219  ndbout << "-----------" << endl;
220  usage();
221  ndbout << endl;
222 
223  SetupTestData();
224 
225  DBA_Open();
226 
227  if(!CreateTable()){
228  DBA_Close();
229  CleanupTestData();
230  return 0;
231  }
232 
233  if(!CreateBindings()){
234  DBA_Close();
235  CleanupTestData();
236  return 0;
237  }
238 
239  CleanTable();
240 
241  sequence(LoopCount);
242 
243  DBA_Close();
244  CleanupTestData();
245 
246  DBA_DestroyBinding(InsertB);
247  DBA_DestroyBinding(UpdateB);
248  DBA_DestroyBinding(DeleteB);
249 }
250 
251 static
252 void
253 ErrorMsg(const char * s){
254  ndbout << s
255  << ": " << DBA_GetLatestError() << "-" << DBA_GetLatestErrorMsg()
256  << ", " << DBA_GetLatestNdbError()
257  << endl;
258 }
259 
260 static
261 int
262 m4(int i){
263  const int j = i - (i & 3);
264  return j;
265 }
266 
267 static
268 void
269 SetupTestData(){
270  ndbout << "Creating testdata" << endl;
271 
272  ColumnDescriptions = new DBA_ColumnDesc_t[NoOfColumns];
273  InsertBindings = new DBA_ColumnBinding_t[NoOfColumns];
274 
275  const int sz = m4((BytesPerInsert - ((NoOfColumns+1)/2)*4)/(NoOfColumns/2));
276  int sum = 0;
277  UpdateBindingColumns = 0;
278  for(int i = 0; i<NoOfColumns; i++){
279  char tmp[16];
280  if((i % 2) == 0){
281  sprintf(tmp, "I%d", i);
282  ColumnDescriptions[i].DataType = DBA_INT;
283  ColumnDescriptions[i].Size = 4;
284  sum += 4;
285  } else {
286  sprintf(tmp, "S%d", i);
287  ColumnDescriptions[i].DataType = DBA_CHAR;
288  ColumnDescriptions[i].Size = sz;
289  sum += sz;
290  }
291  ColumnDescriptions[i].IsKey = 0;
292  ColumnDescriptions[i].Name = strdup(tmp);
293 
294  InsertBindings[i].Name = strdup(tmp);
295  InsertBindings[i].DataType = ColumnDescriptions[i].DataType;
296  InsertBindings[i].Size = ColumnDescriptions[i].Size;
297  InsertBindings[i].Offset = sum - ColumnDescriptions[i].Size;
298  InsertBindings[i].Ptr = 0;
299 
300  if(sum <= BytesPerUpdate)
301  UpdateBindingColumns++;
302  }
303  if(UpdateBindingColumns == 1)
304  UpdateBindingColumns++;
305 
306  ColumnDescriptions[0].IsKey = 1;
307 
308  assert(sum <= BytesPerInsert);
309  sprintf(TableName, "NEWTON_%d_%d", sum, NoOfColumns);
310 
311  UpdateBindings = new DBA_ColumnBinding_t[UpdateBindingColumns];
312  memcpy(UpdateBindings, InsertBindings,
313  UpdateBindingColumns*sizeof(DBA_ColumnBinding_t));
314 
315  DeleteBindings = new DBA_ColumnBinding_t[1];
316  memcpy(DeleteBindings, InsertBindings,
317  1*sizeof(DBA_ColumnBinding_t));
318 
319  TestData = (char *)malloc(NoOfTransactions *
320  OperationsPerTransaction * BytesPerInsert);
321 
322  assert(TestData != 0);
323  for(int i = 0; i<NoOfTransactions; i++)
324  for(int j = 0; j<OperationsPerTransaction; j++){
325  const int pk = i * OperationsPerTransaction + j;
326  setPK(pk, pk);
327  }
328 }
329 
330 static
331 void
332 CleanupTestData(){
333  free(TestData);
334  for(int i = 0; i<NoOfColumns; i++){
335  free((char*)ColumnDescriptions[i].Name);
336  free((char*)InsertBindings[i].Name);
337  }
338  delete [] ColumnDescriptions;
339  delete [] InsertBindings;
340  delete [] UpdateBindings;
341  delete [] DeleteBindings;
342 }
343 
344 
345 static bool CleanReturnValue = true;
346 static int CleanCallbacks = 0;
347 static int CleanRows = 0;
348 
349 extern "C"
350 void
351 CleanCallback(DBA_ReqId_t reqId, DBA_Error_t error, DBA_ErrorCode_t ec){
352  CleanCallbacks++;
353  if(error == DBA_NO_ERROR)
354  CleanRows++;
355 }
356 
357 static
358 bool
359 CleanTable(){
360  ndbout << "Cleaning table..." << flush;
361  CleanReturnValue = true;
362  CleanCallbacks = 0;
363  CleanRows = 0;
364  for(int i = 0; i<NoOfTransactions * OperationsPerTransaction; i++){
365  DBA_ArrayDeleteRows(DeleteB,
366  getPtr(i), 1,
367  CleanCallback);
368  while((i-CleanCallbacks)>ParallellTransactions)
369  NdbSleep_MilliSleep(100);
370  }
371  while(CleanCallbacks != (NoOfTransactions * OperationsPerTransaction))
372  NdbSleep_SecSleep(1);
373 
374  ndbout << CleanRows << " rows deleted" << endl;
375 
376  return CleanReturnValue;
377 }
378 
379 static
380 bool
381 CreateBindings(){
382  ndbout << "Creating bindings" << endl;
383  InsertB = UpdateB = DeleteB = 0;
384 
385  InsertB = DBA_CreateBinding(TableName, NoOfColumns,
386  InsertBindings, BytesPerInsert);
387  if(InsertB == 0){
388  ErrorMsg("Failed to create insert bindings");
389  return false;
390  }
391 
392  UpdateB = DBA_CreateBinding(TableName, UpdateBindingColumns,
393  UpdateBindings, BytesPerInsert);
394  if(UpdateB == 0){
395  ErrorMsg("Failed to create update bindings");
396  DBA_DestroyBinding(InsertB);
397  return false;
398  }
399 
400  DeleteB = DBA_CreateBinding(TableName, 1,
401  DeleteBindings, BytesPerInsert);
402  if(DeleteB == 0){
403  ErrorMsg("Failed to create delete bindings");
404  DBA_DestroyBinding(InsertB);
405  DBA_DestroyBinding(UpdateB);
406  return false;
407  }
408  return true;
409 }
410 
411 static
412 bool
413 CreateTable(){
414  ndbout << "Creating " << TableName << endl;
415  return DBA_CreateTable( TableName,
416  NoOfColumns,
417  ColumnDescriptions ) == DBA_NO_ERROR;
418 }
419 
423 static NdbTimer SequenceTimer;
424 
425 static int CurrentOp = NP_Insert;
426 static int SequenceSent = 0;
427 static int SequenceRecv = 0;
428 static NDBT_Stats SequenceStats[NP_MAX][4];
429 static NDBT_Stats SequenceLatency[NP_MAX];
430 
431 static int HashMax;
432 static DBA_ReqId_t * ReqHash; // ReqId - Latency/Row
433 static int * ReqHashPos; // (row in StartTime)
434 
435 static int SequenceLatencyPos;
436 static NDB_TICKS * StartTime;
437 
438 static
439 inline
440 int
441 computeHashMax(int elements){
442  HashMax = 1;
443  while(HashMax < elements)
444  HashMax *= 2;
445 
446  if(HashMax < 1024)
447  HashMax = 1024;
448 
449  return HashMax;
450 }
451 
452 static
453 inline
454 int
455 hash(DBA_ReqId_t request){
456  int r = (request >> 2) & (HashMax-1);
457  return r;
458 }
459 
460 static
461 inline
462 void
463 addRequest(DBA_ReqId_t request, int pos){
464 
465  int i = hash(request);
466 
467  while(ReqHash[i] != 0)
468  i = ((i + 1) & (HashMax-1));
469 
470  ReqHash[i] = request;
471  ReqHashPos[i] = pos;
472 }
473 
474 static
475 inline
476 int
477 getRequest(DBA_ReqId_t request){
478 
479  int i = hash(request);
480 
481  while(ReqHash[i] != request)
482  i = ((i + 1) & (HashMax-1));
483 
484  ReqHash[i] = 0;
485 
486  return ReqHashPos[i];
487 }
488 
489 extern "C"
490 void
491 SequenceCallback(DBA_ReqId_t reqId, DBA_Error_t error, DBA_ErrorCode_t ec){
492  int p = getRequest(reqId) - 1;
493 
494  if(error != DBA_NO_ERROR){
495  ndbout << "p = " << p << endl;
496  ndbout << "DBA_GetErrorMsg(" << error << ") = "
497  << DBA_GetErrorMsg(error) << endl;
498  ndbout << "DBA_GetNdbErrorMsg(" << ec << ") = "
499  << DBA_GetNdbErrorMsg(ec) << endl;
500 
501  assert(error == DBA_NO_ERROR);
502  }
503 
504  SequenceRecv++;
505  if(SequenceRecv == NoOfTransactions){
506  SequenceTimer.doStop();
507  }
508 
509  if((p & 127) == 127){
510  NDB_TICKS t = NdbTick_CurrentMillisecond() - StartTime[p];
511  SequenceLatency[CurrentOp].addObservation(t);
512  }
513 }
514 
515 typedef DBA_ReqId_t (* DBA_ArrayFunction)( const DBA_Binding_t* pBindings,
516  const void * pData,
517  int NbRows,
518  DBA_AsyncCallbackFn_t CbFunc );
519 
520 inline
521 int
522 min(int a, int b){
523  return a > b ? b : a;
524 }
525 
526 static
527 void
528 SequenceOp(DBA_ArrayFunction func, const DBA_Binding_t* pBindings, int op){
529  SequenceSent = 0;
530  SequenceRecv = 0;
531  SequenceLatencyPos = 1;
532  CurrentOp = op;
533 
534  SequenceTimer.doStart();
535  for(int i = 0; i<NoOfTransactions; ){
536  const int l1 = ParallellTransactions - (SequenceSent - SequenceRecv);
537  const int l2 = min(NoOfTransactions - i, l1);
538  for(int j = 0; j<l2; j++){
539  const DBA_ReqId_t r = func(pBindings,
540  getPtr(i*OperationsPerTransaction),
541  OperationsPerTransaction,
542  SequenceCallback);
543  assert(r != 0);
544  SequenceSent++;
545  addRequest(r, i + 1);
546  i++;
547 
548  if((SequenceSent & 127) == 127){
549  NDB_TICKS t = NdbTick_CurrentMillisecond();
550  StartTime[i] = t;
551  }
552  }
553  if(l2 == 0)
554  NdbSleep_MilliSleep(10);
555  }
556 
557  while(SequenceRecv != SequenceSent)
558  NdbSleep_SecSleep(1);
559 
560  ndbout << "Performed " << NoOfTransactions << " " << Operations[op]
561  << " in ";
562 
563  double p = NoOfTransactions * 1000;
564  double t = SequenceTimer.elapsedTime();
565  double o = p * OperationsPerTransaction;
566 
567  p /= t;
568  o /= t;
569 
570  int _p = p;
571  int _o = o;
572 
573  double b = 0;
574 
575  switch(op){
576  case NP_Insert:
577  case NP_WriteInsert:
578  case NP_WriteUpdate:
579  case NP_BulkRead:
580  b = BytesPerInsert;
581  break;
582  case NP_Update:
583  b = BytesPerUpdate;
584  break;
585  case NP_Delete:
586  b = 4;
587  break;
588  default:
589  b = 0;
590  }
591  b *= NoOfTransactions * OperationsPerTransaction;
592  b /= t;
593  int _b = b;
594 
595  SequenceStats[op][0].addObservation(t);
596  SequenceStats[op][1].addObservation(p);
597  SequenceStats[op][2].addObservation(o);
598  SequenceStats[op][3].addObservation(b);
599 
600  int t2 = SequenceStats[op][0].getMean();
601  int p2 = SequenceStats[op][1].getMean();
602  int o2 = SequenceStats[op][2].getMean();
603  int b2 = SequenceStats[op][3].getMean();
604 
605  ndbout << SequenceTimer.elapsedTime() << "(" << t2 << ")ms";
606  ndbout << " -> " << _p << "(" << p2 << ") T/s - " << _o
607  << "(" << o2 << ") O/s - " << _b << "(" << b2 << ") Kb/s" << endl;
608 
609  ndbout << " Latency (ms) Avg: " << (int)SequenceLatency[op].getMean()
610  << " min: " << (int)SequenceLatency[op].getMin()
611  << " max: " << (int)SequenceLatency[op].getMax()
612  << " stddev: " << (int)SequenceLatency[op].getStddev()
613  << " n: " << SequenceLatency[op].getCount() << endl;
614 }
615 
619 static
620 void
621 sequence(int loops){
622  computeHashMax(ParallellTransactions);
623  ReqHash = new DBA_ReqId_t[HashMax];
624  ReqHashPos = new int[HashMax];
625  StartTime = new NDB_TICKS[NoOfTransactions];
626 
627  for(int i = 0; i<NP_MAX; i++){
628  SequenceLatency[i].reset();
629  for(int j = 0; j<4; j++)
630  SequenceStats[i][j].reset();
631  }
632  for(int i = 0; i<loops; i++){
633  ndbout << "Loop #" << (i+1) << endl;
634  SequenceOp(DBA_ArrayInsertRows, InsertB, NP_Insert);
635 
636  // BulkRead
637 
638  SequenceOp(DBA_ArrayUpdateRows, UpdateB, NP_Update);
639  SequenceOp(DBA_ArrayWriteRows, InsertB, NP_WriteUpdate);
640  SequenceOp(DBA_ArrayDeleteRows, DeleteB, NP_Delete);
641  SequenceOp(DBA_ArrayWriteRows, InsertB, NP_WriteInsert);
642  SequenceOp(DBA_ArrayDeleteRows, DeleteB, NP_Delete);
643  ndbout << "-------------------" << endl << endl;
644  }
645 
646  delete [] ReqHash;
647  delete [] ReqHashPos;
648  delete [] StartTime;
649 }