00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031 #include <stdio.h>
00032 #include <time.h>
00033 #include <stdlib.h>
00034 #include <fcntl.h>
00035
00036 #include <ctype.h>
00037
00038 #ifdef WIN32
00039 #include <conio.h>
00040 #else
00041 #endif
00042
00043 #include "CompressMan.hpp"
00044 #include "VPathExprMan.hpp"
00045 #include "Input.hpp"
00046 #include "CurPath.hpp"
00047
00048
00049 #ifdef XMILL
00050 #include "Output.hpp"
00051 #include "XMLParse.hpp"
00052 #include "SAXClient.hpp"
00053 #include "Compress.hpp"
00054 #include "ContMan.hpp"
00055 #endif
00056
00057 #ifdef XDEMILL
00058 #include "LabelDict.hpp"
00059 #include "XMLOutput.hpp"
00060 #include "SmallUncompress.hpp"
00061 #include "UnCompCont.hpp"
00062 #endif
00063
00064 #define MAGIC_KEY 0x5e3d29e
00065
00066
00067
00068 CurPath curpath;
00069 LabelDict globallabeldict;
00070
00071 VPathExprMan pathexprman;
00072
00073 #ifdef XMILL
00074 PathTree pathtree;
00075
00076 CompressContainerMan compresscontman;
00077
00078
00079
00080
00081 CompressContainerBlock *globalcontblock;
00082
00083
00084
00085 CompressContainer *globalwhitespacecont;
00086 CompressContainer *globalspecialcont;
00087 CompressContainer *globaltreecont;
00088
00089
00090
00091 extern unsigned long fileheadersize_orig;
00092 extern unsigned long fileheadersize_compressed;
00093 #endif
00094
00095
00096
00097 #ifdef XDEMILL
00098
00099 XMLOutput output;
00100
00101
00102 extern UncompressContainerMan uncomprcont;
00103
00104
00105
00106
00107
00108
00109
00110
00111 unsigned char *memoryalloc_buf=NULL;
00112 unsigned char *memoryalloc_curptr=NULL;
00113 unsigned long memoryalloc_bufsize=0;
00114
00115 #endif
00116
00117
00118 extern char usestdout;
00119 extern char no_output;
00120 #ifdef TIMING
00121 extern char timing;
00122 #endif
00123 extern char globalfullwhitespacescompress;
00124 extern char verbose;
00125 extern char output_initialized;
00126 extern char delete_inputfiles;
00127 extern unsigned long memory_cutoff;
00128
00129
00130
00131
00132 MemStreamer tmpmem(5);
00133 MemStreamer mainmem(1000);
00134
00135 MemStreamer blockmem(1000);
00136
00137
00138
00139
00140
00141 #ifdef XMILL
00142 void FSMInit();
00143
00144
00145 void InitSpecialContainerSizeSum();
00146 void PrintSpecialContainerSizeSum();
00147
00148 void Compress(char *srcfile,char *destfile);
00149 #endif
00150
00151 #ifdef XDEMILL
00152 void Uncompress(char *sourcefile,char *destfile);
00153 #endif
00154
00155
00156 void PrintUsage(char showmoreoptions);
00157 int HandleAllOptions(char **argv,int argc);
00158
00159
00160
00161
00162 extern char overwrite_files;
00163 extern char skip_all_files;
00164
00165 char AskOverwriteFile(char *file)
00166
00167
00168
00169
00170 {
00171 int c;
00172
00173 if((!FileExists(file))||(overwrite_files))
00174
00175
00176 return 1;
00177
00178 printf("Overwrite file %s ? (Y(es) | N(o) | A(ll) | Q(uit)) ",file);
00179
00180
00181 do
00182 {
00183 #ifdef WIN32
00184 c=_getch();
00185 printf("\n");
00186 #else
00187 c=getchar();
00188 #endif
00189 switch(toupper(c))
00190 {
00191 case 'Y': return 1;
00192 case 'N': return 0;
00193 case 'Q': skip_all_files=1;return 0;
00194 case 'A': overwrite_files=1;return 1;
00195 }
00196 }
00197 while(1);
00198 }
00199
00200 void HandleSingleFile(char *file)
00201
00202
00203
00204 {
00205 int len=strlen(file);
00206 char *outfilename=file+len+5;
00207
00208
00209
00210 strcpy(outfilename,file);
00211
00212 try{
00213
00214 #ifdef XMILL
00215
00216
00217
00218 if((len>=4)&&(strcmp(file+len-4,".xml")==0))
00219 strcpy(outfilename+len-4,".xmi");
00220 else
00221 strcat(outfilename,".xm");
00222
00223 Compress(file,usestdout ? NULL : outfilename);
00224
00225 #ifdef PROFILE
00226 if(verbose)
00227 globallabeldict.PrintProfile();
00228 #endif
00229
00230 #endif
00231
00232 #ifdef XDEMILL
00233
00234
00235 if((len>=3)&&(strcmp(file+len-3,".xm")==0))
00236
00237 {
00238 outfilename[len-3]=0;
00239 Uncompress(file,usestdout ? NULL : outfilename);
00240 }
00241 else
00242 {
00243
00244 if((len>=4)&&(strcmp(file+len-4,".xmi")==0))
00245 {
00246 strcpy(outfilename+len-4,".xml");
00247 Uncompress(file,usestdout ? NULL : outfilename);
00248 }
00249 else
00250 {
00251
00252
00253
00254
00255 if((len>=4)&&(strcmp(file+len-4,".xml")==0))
00256 {
00257 strcpy(file+len-4,".xmi");
00258 if(FileExists(file))
00259 {
00260 Uncompress(file,usestdout ? NULL : outfilename);
00261 return;
00262 }
00263 strcpy(file+len-4,".xml");
00264 }
00265
00266
00267 strcpy(file+len,".xm");
00268
00269 if(FileExists(file)==0)
00270 {
00271 strcpy(file+len,"");
00272 Error("Could not find file '");
00273 ErrorCont(file);
00274 ErrorCont("' with extension '.xm'!");
00275 PrintErrorMsg();
00276 return;
00277 }
00278 Uncompress(file,usestdout ? NULL : outfilename);
00279 return;
00280 }
00281 }
00282 #endif
00283 }
00284 catch(XMillException *)
00285
00286 {
00287 Error("Error in file '");
00288 ErrorCont(file);
00289 ErrorCont("':");
00290 PrintErrorMsg();
00291 }
00292 }
00293
00294 void HandleFileArg(char *filepattern)
00295
00296
00297
00298
00299 {
00300 char fullpath[400];
00301 #ifdef WIN32
00302 _finddata_t finddata;
00303 long handle;
00304 char *ptr;
00305 int fullpathlen;
00306
00307
00308
00309
00310 ptr=filepattern;
00311 while(*ptr!=0)
00312 {
00313 if((*ptr=='*')||(*ptr=='?'))
00314 break;
00315 ptr++;
00316 }
00317
00318 if(*ptr==0)
00319
00320 {
00321 strcpy(fullpath,filepattern);
00322 HandleSingleFile(fullpath);
00323 return;
00324 }
00325
00326
00327
00328
00329 ptr=filepattern+strlen(filepattern)-1;
00330
00331 while(ptr>=filepattern)
00332 {
00333 if((*ptr=='\\')||(*ptr=='/'))
00334 break;
00335 ptr--;
00336 }
00337
00338 if(ptr<filepattern)
00339 {
00340
00341 *fullpath=0;
00342 fullpathlen=0;
00343 }
00344 else
00345 {
00346
00347
00348 memcpy(fullpath,filepattern,ptr-filepattern+1);
00349 fullpath[ptr-filepattern+1]=0;
00350 fullpathlen=ptr-filepattern+1;
00351 }
00352
00353
00354 handle=_findfirst(filepattern,&finddata);
00355 if(handle==-1)
00356 {
00357 printf("Could not find %s!\n",filepattern);
00358 return;
00359 }
00360
00361 do
00362 {
00363
00364 strcpy(fullpath+fullpathlen,finddata.name);
00365
00366 HandleSingleFile(fullpath);
00367 if(skip_all_files)
00368 break;
00369
00370 if(_findnext(handle,&finddata)!=0)
00371 break;
00372 }
00373 while(1);
00374
00375 _findclose(handle);
00376 #else
00377
00378
00379
00380 strcpy(fullpath,filepattern);
00381 HandleSingleFile(fullpath);
00382 #endif
00383 }
00384
00385
00386
00387
00388 #ifdef WIN32
00389 int _cdecl main(int argc,char **argv)
00390 #else
00391 int main(int argc,char **argv)
00392 #endif
00393 {
00394 int fileidx;
00395
00396
00397 #ifdef WIN32
00398 _fmode=_O_BINARY;
00399 #endif
00400
00401 if(argc==1)
00402 {
00403 PrintUsage(0);
00404 return 0;
00405 }
00406
00407 #ifdef XMILL
00408 else
00409 {
00410 if((argc==2)&&(strcmp(argv[1],"-h")==0))
00411
00412 {
00413 PrintUsage(1);
00414 return 0;
00415 }
00416 }
00417 #endif
00418
00419
00420
00421 try{
00422
00423 globallabeldict.Init();
00424
00425 #ifdef XMILL
00426
00427
00428 FSMInit();
00429
00430 #ifdef USE_FORWARD_DATAGUIDE
00431 extern void InitForwardDataGuide();
00432
00433 InitForwardDataGuide();
00434 #endif
00435
00436 #endif
00437
00438
00439 fileidx=HandleAllOptions(argv+1,argc-1)+1;
00440
00441 #ifdef XMILL
00442
00443
00444 char *pathptr="//#";
00445 pathexprman.AddNewVPathExpr(pathptr,pathptr+strlen(pathptr));
00446 pathptr="/";
00447 pathexprman.AddNewVPathExpr(pathptr,pathptr+strlen(pathptr));
00448
00449 globallabeldict.FinishedPredefinedLabels();
00450
00451
00452
00453
00454 pathexprman.InitWhitespaceHandling();
00455
00456
00457
00458
00459
00460
00461 #endif
00462
00463
00464 if(fileidx>=argc)
00465 {
00466 if(usestdout)
00467 {
00468 #ifdef XMILL
00469 Compress(NULL,NULL);
00470 #endif
00471 #ifdef XDEMILL
00472 Uncompress(NULL,NULL);
00473 #endif
00474 return 0;
00475 }
00476 else
00477 {
00478 Error("No input file specified! Specify '-c' to use stdin/stdout");
00479 Exit();
00480 }
00481 }
00482
00483 }
00484 catch(XMillException *)
00485
00486 {
00487 return -1;
00488 }
00489
00490
00491 do
00492 {
00493 HandleFileArg(argv[fileidx]);
00494 if(skip_all_files)
00495 break;
00496 fileidx++;
00497 }
00498 while(fileidx<argc);
00499
00500 return 0;
00501 }
00502
00503
00504
00505
00506
00507
00508
00509
00510 #ifdef XMILL
00511
00512 inline void StoreFileHeader(Compressor *compressor)
00513 {
00514 MemStreamer tmpoutputstream(1);
00515
00516 tmpoutputstream.StoreSInt32(
00517 (globalfullwhitespacescompress==WHITESPACE_IGNORE) ? 1 : 0,
00518 MAGIC_KEY);
00519
00520 pathexprman.Store(&tmpoutputstream);
00521
00522 compressor->CompressMemStream(&tmpoutputstream);
00523 }
00524
00525 inline void CompressBlockHeader(Compressor *compressor,unsigned long totaldatasize)
00526 {
00527 MemStreamer memstream;
00528
00529 memstream.StoreUInt32(totaldatasize);
00530
00531
00532
00533 compresscontman.StoreMainInfo(&memstream);
00534
00535 compressor->CompressMemStream(&memstream);
00536
00537
00538 globallabeldict.Store(compressor);
00539
00540 compressman.CompressSmallGlobalData(compressor);
00541
00542 compresscontman.CompressSmallContainers(compressor);
00543 }
00544
00545 char fileheader_iswritten=0;
00546
00547 inline void CompressCurrentBlock(Output *output,unsigned long totaldatasize)
00548 {
00549 {
00550 Compressor compressor(output);
00551 unsigned long headersize,headersize_compressed;
00552
00553 if(fileheader_iswritten==0)
00554 {
00555 StoreFileHeader(&compressor);
00556 fileheader_iswritten=1;
00557 }
00558 CompressBlockHeader(&compressor,totaldatasize);
00559 compressor.FinishCompress(&headersize,&headersize_compressed);
00560
00561 fileheadersize_orig +=headersize;
00562 fileheadersize_compressed +=headersize_compressed;
00563 }
00564
00565 compressman.CompressLargeGlobalData(output);
00566
00567
00568 compresscontman.CompressLargeContainers(output);
00569 }
00570
00571 void Compress(char *srcfile,char *destfile)
00572 {
00573 SAXClient saxclient;
00574 XMLParse xmlparse;
00575 Output output;
00576 #ifdef TIMING
00577 clock_t c1,c2,c3;
00578 #endif
00579 int parsetime=0,compresstime=0;
00580 char isend;
00581 unsigned long totaldatasize;
00582
00583
00584
00585
00586 InitSpecialContainerSizeSum();
00587
00588 fileheader_iswritten=0;
00589
00590 if(AskOverwriteFile(destfile)==0)
00591 return;
00592
00593 if(xmlparse.OpenFile(srcfile)==0)
00594 {
00595 Error("Could not find file '");
00596 ErrorCont(srcfile);
00597 ErrorCont("'!");
00598 PrintErrorMsg();
00599 return;
00600 }
00601
00602 if(output.CreateFile((no_output==0) ? destfile : "")==0)
00603 {
00604 Error("Could not create output file '");
00605 ErrorCont(destfile);
00606 PrintErrorMsg();
00607 xmlparse.CloseFile();
00608 return;
00609 }
00610
00611 mainmem.StartNewMemBlock();
00612
00613 #ifdef USE_FORWARD_DATAGUIDE
00614 pathdict.Init();
00615 pathtree.CreateRootNode();
00616 #endif
00617
00618
00619 try{
00620 do
00621 {
00622 #ifndef USE_FORWARD_DATAGUIDE
00623 pathdict.Init();
00624 pathtree.CreateRootNode();
00625 #else
00626 pathdict.ResetContBlockPtrs();
00627 #endif
00628
00629 globalcontblock =compresscontman.CreateNewContainerBlock(3,0,NULL,NULL);
00630 globaltreecont =globalcontblock->GetContainer(0);
00631 globalwhitespacecont =globalcontblock->GetContainer(1);
00632 globalspecialcont =globalcontblock->GetContainer(2);
00633 #ifdef TIMING
00634 if(timing)
00635 c1=clock();
00636 #endif
00637
00638 isend=xmlparse.DoParsing(&saxclient);
00639 if(isend)
00640 isend=1;
00641
00642 #ifdef TIMING
00643 if(timing)
00644 c2=clock();
00645 #endif
00646
00647 compresscontman.FinishCompress();
00648
00649 totaldatasize= compresscontman.GetDataSize()+
00650 compressman.GetDataSize();
00651
00652 CompressCurrentBlock(&output,totaldatasize);
00653 #ifdef TIMING
00654 if(timing)
00655 {
00656 c3=clock();
00657 parsetime+=(c2-c1);
00658 compresstime+=(c3-c2);
00659 }
00660 #endif
00661 #ifdef PROFILE
00662 if(verbose)
00663 printf("Pathtree size: %lu\n",pathtreemem.GetSize());
00664 #endif
00665
00666 if(verbose)
00667 {
00668
00669 #ifdef PROFILE
00670 pathtree.PrintProfile();
00671 pathdict.PrintProfile();
00672 #endif
00673 }
00674
00675 #ifndef USE_FORWARD_DATAGUIDE
00676 pathtree.ReleaseMemory();
00677 #endif
00678 compresscontman.ReleaseMemory();
00679 blockmem.ReleaseMemory(1000);
00680
00681
00682
00683
00684
00685 }
00686 while(isend==0);
00687 }
00688 catch(XMillException *)
00689 {
00690 output.CloseAndDeleteFile();
00691 xmlparse.CloseFile();
00692 Exit();
00693 }
00694 #ifdef TIMING
00695 if(timing)
00696 printf("%fs + %fs = %fs\n",(float)parsetime/(float)CLOCKS_PER_SEC,
00697 (float)compresstime/(float)CLOCKS_PER_SEC,
00698 (float)(parsetime+compresstime)/(float)CLOCKS_PER_SEC);
00699 #endif
00700 if(verbose)
00701 PrintSpecialContainerSizeSum();
00702
00703 #ifdef PROFILE
00704 if(verbose)
00705 curpath.PrintProfile();
00706 #endif
00707
00708 xmlparse.CloseFile();
00709 output.CloseFile();
00710
00711 globallabeldict.Reset();
00712
00713 mainmem.RemoveLastMemBlock();
00714
00715 if(delete_inputfiles)
00716 RemoveFile(srcfile);
00717 }
00718
00719 #endif //XMILL
00720
00721
00722
00723
00724
00725
00726
00727 #ifdef XDEMILL
00728
00729 void DecodeTreeBlock(UncompressContainer *treecont,UncompressContainer *whitespacecont,UncompressContainer *specialcont,XMLOutput *output);
00730
00731
00732
00733 void UncompressFileHeader(SmallBlockUncompressor *uncompressor)
00734 {
00735 char iswhitespaceignore;
00736
00737 if(uncompressor->LoadSInt32(&iswhitespaceignore)!=MAGIC_KEY)
00738 {
00739 Error("The file is not a compressed XMill file!");
00740 Exit();
00741 }
00742
00743 if(iswhitespaceignore)
00744 {
00745 globalfullwhitespacescompress=WHITESPACE_IGNORE;
00746 if(output_initialized==0)
00747 output.Init(XMLINTENT_SPACES,0,1);
00748 }
00749 else
00750 {
00751 globalfullwhitespacescompress=WHITESPACE_STOREGLOBAL;
00752 if(output_initialized==0)
00753 output.Init(XMLINTENT_NONE,0,1);
00754 }
00755 pathexprman.Load(uncompressor);
00756 }
00757
00758 static char fileheader_isread=0;
00759
00760 char UncompressBlockHeader(Input *input)
00761 {
00762 SmallBlockUncompressor uncompressor(input);
00763
00764 if(fileheader_isread==0)
00765 {
00766 UncompressFileHeader(&uncompressor);
00767 fileheader_isread=1;
00768 }
00769 else
00770 {
00771 if(input->IsEndOfFile())
00772 return 1;
00773 }
00774
00775 memory_cutoff=uncompressor.LoadUInt32();
00776
00777 SetMemoryAllocationSize(memory_cutoff);
00778
00779 uncomprcont.Load(&uncompressor);
00780
00781 globallabeldict.Load(&uncompressor);
00782
00783 compressman.UncompressSmallGlobalData(&uncompressor);
00784
00785 uncomprcont.AllocateContMem();
00786
00787 uncomprcont.UncompressSmallContainers(&uncompressor);
00788
00789 return 0;
00790 }
00791
00792 #undef CreateFile
00793
00794 void Uncompress(char *sourcefile,char *destfile)
00795
00796 {
00797 Input input;
00798 #ifdef TIMING
00799 clock_t c1,c2,c3,ct1=0,ct2=0;
00800 #endif
00801
00802 UncompressContainer *uncomprtreecont;
00803 UncompressContainer *uncomprwhitespacecont;
00804 UncompressContainer *uncomprspecialcont;
00805
00806 if(AskOverwriteFile(destfile)==0)
00807 return;
00808
00809 if(input.OpenFile(sourcefile)==0)
00810 {
00811 Error("Could not find file '");
00812 ErrorCont(sourcefile);
00813 PrintErrorMsg();
00814 return;
00815 }
00816
00817 globallabeldict.Init();
00818
00819 if(output.CreateFile((no_output==0) ? destfile : "")==0)
00820 {
00821 Error("Could not create output file '");
00822 ErrorCont(destfile);
00823 PrintErrorMsg();
00824 input.CloseFile();
00825 return;
00826 }
00827 #ifdef TIMING
00828 c1=clock();
00829 #endif
00830
00831 unsigned long blockidx=0;
00832
00833 mainmem.StartNewMemBlock();
00834
00835 while(UncompressBlockHeader(&input)==0)
00836 {
00837 compressman.UncompressLargeGlobalData(&input);
00838 uncomprcont.UncompressLargeContainers(&input);
00839
00840 uncomprcont.Init();
00841
00842 uncomprtreecont =uncomprcont.GetContBlock(0)->GetContainer(0);
00843 uncomprwhitespacecont=uncomprcont.GetContBlock(0)->GetContainer(1);
00844 uncomprspecialcont =uncomprcont.GetContBlock(0)->GetContainer(2);
00845 #ifdef TIMING
00846 c2=clock();
00847 #endif
00848
00849 DecodeTreeBlock(uncomprtreecont,uncomprwhitespacecont,uncomprspecialcont,&output);
00850 #ifdef TIMING
00851 c3=clock();
00852 #endif
00853
00854
00855
00856
00857
00858
00859
00860 #ifdef TIMING
00861 ct1+=c2-c1;
00862 ct2+=c3-c2;
00863 #endif
00864
00865 uncomprcont.FinishUncompress();
00866 uncomprcont.ReleaseContMem();
00867 compressman.FinishUncompress();
00868 blockmem.ReleaseMemory(1000);
00869 #ifdef TIMING
00870 c1=clock();
00871 #endif
00872 blockidx++;
00873 }
00874 #ifdef TIMING
00875 if(verbose)
00876 printf("%fs + %fs = %fs\n",(float)ct1/(float)CLOCKS_PER_SEC,
00877 (float)ct2/(float)CLOCKS_PER_SEC,
00878 (float)(ct1+ct2)/(float)CLOCKS_PER_SEC);
00879 #endif
00880 input.CloseFile();
00881 output.CloseFile();
00882
00883 if(delete_inputfiles)
00884 RemoveFile(sourcefile);
00885
00886 globallabeldict.Reset();
00887
00888 fileheader_isread=0;
00889
00890 mainmem.RemoveLastMemBlock();
00891 }
00892
00893 #endif // XDEMILL
00894
00895
00896
00897
00898
00899
00900
00901
00902
00903
00904
00905
00906
00907
00908