00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036 #ifndef LABELDICT_HPP
00037 #define LABELDICT_HPP
00038
00039
00040 #define MAXLABEL_NUM 65535L
00041
00042 #include "Types.hpp"
00043 #include "MemStreamer.hpp"
00044 #include "Load.hpp"
00045
00046 #ifdef XMILL
00047 #include "Compress.hpp"
00048 #endif
00049
00050 #ifdef XDEMILL
00051 #include "SmallUncompress.hpp"
00052 #endif
00053
00054 #define ISATTRIB(labelid) (((labelid)&ATTRIBLABEL_STARTIDX)?1:0)
00055
00056 extern MemStreamer mainmem;
00057
00058 #ifdef XMILL
00059
00060
00061
00062
00063 #define HASHTABLE_SIZE 256
00064 #define HASHTABLE_MASK 255
00065 #define HASHTABLE_SHIFT 8
00066 #define HASHTABLE_SHIFT2 16
00067 #define HASHTABLE_SHIFT3 24
00068
00069
00070
00071
00072
00073 struct CompressLabelDictItem
00074
00075 {
00076 unsigned short len;
00077 TLabelID labelid;
00078 CompressLabelDictItem *nextsamehash;
00079 CompressLabelDictItem *next;
00080
00081 char IsAttrib() { return ISATTRIB(labelid); }
00082
00083 char *GetLabelPtr() { return (char *)(this+1); }
00084
00085
00086 unsigned short GetLabelLen() { return len; }
00087
00088 };
00089 #endif
00090
00091
00092
00093 #ifdef XDEMILL
00094
00095
00096
00097
00098 struct UncompressLabelDictItem
00099
00100 {
00101 unsigned short len;
00102 unsigned char isattrib;
00103 char *strptr;
00104 };
00105
00106 struct UncompressLabelDict
00107
00108
00109
00110 {
00111 TLabelID labelnum;
00112 UncompressLabelDict *next;
00113
00114 UncompressLabelDictItem* GetArrayPtr()
00115
00116 {
00117 return (UncompressLabelDictItem *)(this+1);
00118 }
00119 };
00120 #endif
00121
00122
00123
00124 class LabelDict
00125
00126 {
00127 TLabelID labelnum;
00128
00129 #ifdef XMILL
00130 TLabelID predefinedlabelnum;
00131
00132
00133
00134 CompressLabelDictItem **hashtable;
00135 CompressLabelDictItem *labels,**labelref;
00136
00137
00138
00139 TLabelID savedlabelnum;
00140 CompressLabelDictItem **savedlabelref;
00141 #endif
00142
00143 #ifdef XDEMILL
00144 UncompressLabelDict *labeldictlist;
00145 UncompressLabelDict *lastlabeldict;
00146 #endif
00147
00148 public:
00149 #ifdef PROFILE
00150 unsigned long lookupcount,hashitercount;
00151 #endif
00152
00153 void Init()
00154 {
00155 #ifdef PROFILE
00156 lookupcount=
00157 hashitercount=0;
00158 #endif
00159
00160 labelnum=0;
00161
00162 #ifdef XMILL
00163
00164 labels=NULL;
00165 labelref=&labels;
00166
00167
00168 savedlabelnum=0;
00169 savedlabelref=&labels;
00170
00171
00172
00173
00174 hashtable=(CompressLabelDictItem **)new char[sizeof(CompressLabelDictItem *)*HASHTABLE_SIZE];
00175 if(hashtable==NULL)
00176 ExitNoMem();
00177
00178 for(int i=0;i<HASHTABLE_SIZE;i++)
00179 hashtable[i]=NULL;
00180 #endif
00181 #ifdef XDEMILL
00182
00183 labeldictlist=NULL;
00184 lastlabeldict=NULL;
00185 #endif
00186 }
00187
00188 void Reset()
00189 {
00190 #ifdef XMILL
00191 int i;
00192 TLabelID labelid;
00193 CompressLabelDictItem **hashtableref;
00194
00195
00196
00197 for(i=0;i<HASHTABLE_SIZE;i++)
00198 {
00199 hashtableref=hashtable+i;
00200 while(*hashtableref!=NULL)
00201 {
00202 if((*hashtableref)->labelid>=predefinedlabelnum)
00203
00204
00205 *hashtableref=(*hashtableref)->nextsamehash;
00206 else
00207
00208 hashtableref=&((*hashtableref)->nextsamehash);
00209 }
00210 }
00211
00212
00213
00214 CompressLabelDictItem **curlabelref=&labels;
00215
00216 for(labelid=0;labelid<predefinedlabelnum;labelid++)
00217 curlabelref=&((*curlabelref)->next);
00218
00219 *curlabelref=NULL;
00220
00221 labelref=curlabelref;
00222 labelnum=predefinedlabelnum;
00223
00224
00225 savedlabelnum=0;
00226 savedlabelref=&labels;
00227 #endif
00228 #ifdef XDEMILL
00229
00230 labelnum=0;
00231 labeldictlist=NULL;
00232 lastlabeldict=NULL;
00233 #endif
00234 }
00235 #ifdef XMILL
00236 void FinishedPredefinedLabels()
00237
00238
00239 {
00240 predefinedlabelnum=labelnum;
00241 }
00242 #endif
00243
00244
00245
00246 #ifdef XMILL
00247
00248 int CalcHashIdx(char *label,int len)
00249
00250 {
00251 int val=0;
00252
00253 while(len>0)
00254 {
00255 val+=*label;
00256 val=(val<<(*label&7))+(val>>(32-(*label&7)));
00257 label++;
00258 len--;
00259 }
00260 return ((val>>HASHTABLE_SHIFT)+(val>>HASHTABLE_SHIFT2)+(val>>HASHTABLE_SHIFT3)+val)&HASHTABLE_MASK;
00261 }
00262
00263 TLabelID FindLabelOrAttrib(char *label,unsigned len,unsigned char isattrib)
00264
00265 {
00266 CompressLabelDictItem **hashref=hashtable+CalcHashIdx(label,len);
00267
00268 #ifdef PROFILE
00269 lookupcount++;
00270 #endif
00271
00272
00273 while(*hashref!=NULL)
00274 {
00275 #ifdef PROFILE
00276 hashitercount++;
00277 #endif
00278 if(((*hashref)->len==len)&&
00279 ((*hashref)->IsAttrib()==isattrib)&&
00280 (mymemcmp((*hashref)->GetLabelPtr(),label,len)==0))
00281
00282 return (*hashref)->labelid;
00283
00284 hashref=&((*hashref)->nextsamehash);
00285 }
00286 return LABEL_UNDEFINED;
00287 }
00288
00289 TLabelID CreateLabelOrAttrib(char *label,unsigned len,unsigned char isattrib)
00290
00291 {
00292
00293 mainmem.WordAlign();
00294 CompressLabelDictItem *item=(CompressLabelDictItem *)mainmem.GetByteBlock(sizeof(CompressLabelDictItem)+len);
00295 mainmem.WordAlign();
00296
00297
00298 item->len=(unsigned short)len;
00299 mymemcpy(item->GetLabelPtr(),label,len);
00300
00301
00302 CompressLabelDictItem **hashref=hashtable+CalcHashIdx(label,len);
00303 item->nextsamehash=*hashref;
00304 *hashref=item;
00305
00306
00307 item->next=NULL;
00308
00309 *labelref=item;
00310 labelref=&(item->next);
00311
00312 item->labelid=labelnum;
00313 labelnum++;
00314
00315 if(isattrib)
00316 item->labelid|=ATTRIBLABEL_STARTIDX;
00317
00318 return item->labelid;
00319 }
00320
00321 TLabelID GetLabelOrAttrib(char *label,unsigned len,unsigned char isattrib)
00322
00323 {
00324 TLabelID labelid=FindLabelOrAttrib(label,len,isattrib);
00325
00326 if(labelid==LABEL_UNDEFINED)
00327 return CreateLabelOrAttrib(label,len,isattrib);
00328 else
00329 return labelid;
00330 }
00331
00332 void Store(Compressor *compressor)
00333
00334
00335 {
00336 MemStreamer mem;
00337
00338
00339
00340 mem.StoreUInt32(labelnum-savedlabelnum);
00341
00342
00343 while(*savedlabelref!=NULL)
00344 {
00345 mem.StoreSInt32(((*savedlabelref)->labelid&ATTRIBLABEL_STARTIDX)?1:0,(*savedlabelref)->GetLabelLen());
00346 mem.StoreData((*savedlabelref)->GetLabelPtr(),(*savedlabelref)->GetLabelLen());
00347 savedlabelref=&((*savedlabelref)->next);
00348 }
00349
00350 compressor->CompressMemStream(&mem);
00351
00352 savedlabelnum=labelnum;
00353 }
00354 #endif
00355
00356
00357
00358
00359
00360 #ifdef XDEMILL
00361
00362 #define LABELDICT_MINLABELNUM 32
00363
00364
00365
00366 void Load(SmallBlockUncompressor *uncompress)
00367
00368
00369 {
00370 UncompressLabelDictItem *dictitemptr;
00371 char isattrib;
00372 unsigned dictlabelnum;
00373
00374
00375 unsigned mylabelnum=uncompress->LoadUInt32();
00376 if(mylabelnum>MAXLABEL_NUM)
00377 ExitCorruptFile();
00378
00379
00380 if(mylabelnum==0)
00381 return;
00382
00383 if(lastlabeldict!=NULL)
00384
00385 {
00386 if(lastlabeldict->labelnum<LABELDICT_MINLABELNUM)
00387
00388
00389 {
00390 dictlabelnum=LABELDICT_MINLABELNUM-lastlabeldict->labelnum;
00391 if(dictlabelnum>mylabelnum)
00392 dictlabelnum=mylabelnum;
00393
00394 mylabelnum-=dictlabelnum;
00395
00396 dictitemptr=lastlabeldict->GetArrayPtr()+lastlabeldict->labelnum;
00397
00398 lastlabeldict->labelnum+=dictlabelnum;
00399 labelnum+=dictlabelnum;
00400
00401
00402
00403
00404 while(dictlabelnum--)
00405 {
00406 dictitemptr->len=(unsigned short)uncompress->LoadSInt32(&isattrib);
00407 dictitemptr->isattrib=isattrib;
00408
00409 dictitemptr->strptr=mainmem.GetByteBlock(dictitemptr->len);
00410
00411 mymemcpy(dictitemptr->strptr,
00412 uncompress->LoadData(dictitemptr->len),
00413 dictitemptr->len);
00414
00415 dictitemptr++;
00416 }
00417 if(mylabelnum==0)
00418 return;
00419 }
00420
00421 lastlabeldict->next=(UncompressLabelDict *)mainmem.GetByteBlock(
00422 sizeof(UncompressLabelDict)+
00423 ((mylabelnum>LABELDICT_MINLABELNUM)? mylabelnum : LABELDICT_MINLABELNUM)*sizeof(UncompressLabelDictItem));
00424 lastlabeldict=lastlabeldict->next;
00425 }
00426 else
00427 labeldictlist=lastlabeldict=(UncompressLabelDict *)mainmem.GetByteBlock(
00428 sizeof(UncompressLabelDict)+
00429 ((mylabelnum>LABELDICT_MINLABELNUM)? mylabelnum : LABELDICT_MINLABELNUM)*sizeof(UncompressLabelDictItem));
00430
00431 lastlabeldict->next=NULL;
00432
00433
00434
00435 lastlabeldict->labelnum=mylabelnum;
00436 labelnum+=mylabelnum;
00437
00438 dictlabelnum=mylabelnum;
00439
00440 dictitemptr=lastlabeldict->GetArrayPtr();
00441
00442
00443
00444
00445 while(dictlabelnum--)
00446 {
00447 dictitemptr->len=(unsigned short)uncompress->LoadSInt32(&isattrib);
00448 dictitemptr->isattrib=isattrib;
00449
00450 dictitemptr->strptr=mainmem.GetByteBlock(dictitemptr->len);
00451 mainmem.WordAlign();
00452
00453 mymemcpy(dictitemptr->strptr,
00454 uncompress->LoadData(dictitemptr->len),
00455 dictitemptr->len);
00456
00457 dictitemptr++;
00458 }
00459 }
00460
00461 unsigned long LookupLabel(TLabelID labelid,char **ptr,unsigned char *isattrib)
00462
00463 {
00464 UncompressLabelDict *labeldict=labeldictlist;
00465
00466
00467 while(labelid>=labeldict->labelnum)
00468 {
00469 labelid-=labeldict->labelnum;
00470 labeldict=labeldict->next;
00471 }
00472 UncompressLabelDictItem *item=labeldict->GetArrayPtr()+labelid;
00473
00474 *isattrib=item->isattrib;
00475 *ptr=item->strptr;
00476 return item->len;
00477 }
00478 #endif
00479
00480
00481
00482
00483 #ifdef XMILL
00484 unsigned long LookupCompressLabel(TLabelID labelid,char **ptr)
00485
00486
00487
00488
00489 {
00490 CompressLabelDictItem *item=labels;
00491
00492 labelid=(labelid&(ATTRIBLABEL_STARTIDX-1));
00493
00494 while(labelid--)
00495 item=item->next;
00496
00497 *ptr=item->GetLabelPtr();
00498 return item->GetLabelLen();
00499 }
00500 #endif
00501
00502 void PrintLabel(TLabelID labelid)
00503
00504 {
00505 char *ptr;
00506 unsigned long len;
00507 unsigned char isattrib=ISATTRIB(labelid);
00508
00509 labelid=(labelid&(ATTRIBLABEL_STARTIDX-1));
00510
00511 #ifdef XMILL
00512 len=LookupCompressLabel(labelid,&ptr);
00513 #endif
00514 #ifdef XDEMILL
00515 len=LookupLabel(labelid,&ptr,&isattrib);
00516 #endif
00517
00518 if(isattrib)
00519 {
00520 printf("@");
00521 fwrite(ptr,len,1,stdout);
00522 }
00523 else
00524 fwrite(ptr,len,1,stdout);
00525 }
00526
00527 void Print()
00528
00529 {
00530 for(unsigned long i=0;i<labelnum;i++)
00531 {
00532 printf("%lu : ",i);
00533 PrintLabel((TLabelID)i);
00534 printf("\n");
00535 }
00536 }
00537
00538 #ifdef PROFILE
00539 void PrintProfile()
00540 {
00541 printf("Labeldict: count=%lu lookupcount=%lu hashitercount=%lu\n",labelnum,lookupcount,hashitercount);
00542 }
00543 #endif
00544 };
00545
00546
00547
00548
00549
00550 extern LabelDict globallabeldict;
00551
00552 #endif