src2/creaImageIOTreeHandlerImageAdder.cpp

   1 #include <creaImageIOTreeHandlerImageAdder.h>
   2 #include <creaImageIOSystem.h>
   3 #include "boost/filesystem.hpp"
   4 #include <boost/filesystem/operations.hpp>
   5 #include <boost/utility.hpp>
   6
   7
   8 namespace fs = boost::filesystem;
   9 using boost::filesystem::path;
  10 using boost::next;
  11 using boost::prior;
  12
  13
  14 using namespace crea;
  15
  16 namespace creaImageIO
  17 {
  18   //====================================================================
  19   // Ctor
  20   TreeHandlerImageAdder::TreeHandlerImageAdder(TreeHandler* tree)
  21     : mTreeHandler(tree)
  22   {
  23   }
  24   // Dtor
  25   TreeHandlerImageAdder::~TreeHandlerImageAdder()
  26   {
  27   }
  28   //====================================================================
  29
  30   //====================================================================
  31   void TreeHandlerImageAdder::ConnectProgressObserver(ProgressCallbackType callback)
  32   {
  33     mProgressSignal.connect(callback);
  34   }
  35   //====================================================================
  36
  37   //=====================================================================
  38   bool TreeHandlerImageAdder::IsHandledFile( const std::string& filename)
  39   {
  40     return (mReader.CanRead(filename));
  41   }
  42   //=====================================================================
  43
  44   //=====================================================================
  45   void TreeHandlerImageAdder::AddFiles( const std::vector<std::string>& filenames)
  46   {
  47     mProgress.Reset();
  48
  49     unsigned int nbf = filenames.size();
  50     std::vector<std::string>::const_iterator i;
  51     for (i=filenames.begin();i!=filenames.end();++i)
  52       {
  53         mTimestampHandler->AddFile((*i), fs::last_write_time(*i), time(0),mCurrentDB);
  54         mProgress.IncNumberScannedFiles();
  55         if (IsHandledFile(*i))
  56           {
  57             mProgress.IncNumberHandledFiles();
  58                 mSynchronizer->InsertAddOp((*i),"0","1",mCurrentDB);
  59                 std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",(*i),mCurrentDB);
  60                 std::stringstream removedOn;
  61                 removedOn<<time(0);
  62                 mSynchronizer->InsertIgnoreFile(addKey,(*i),"0",removedOn.str(),mCurrentDB);
  63             AddFile(*i);
  64           }
  65         mProgressSignal(mProgress);
  66         if (mProgress.GetStop()) break;
  67       }
  68   }
  69   //=====================================================================
  70
  71   //=====================================================================
  72   void TreeHandlerImageAdder::AddDirectory( const std::string& directory,
  73                                             bool recurse)
  74   {
  75     mProgress.Reset();
  76         std::stringstream files;
  77
  78         std::stringstream rec;
  79         rec<<recurse;
  80         mSynchronizer->InsertAddOp(directory,rec.str(),"0",mCurrentDB);
  81         std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",directory,mCurrentDB);
  82         mTreeHandler->BeginTransaction();
  83         AddDirectoryRecursor( directory, recurse, addKey );
  84
  85         int nFiles=GetProgress().GetNumberAddedFiles();
  86         files<<nFiles;
  87         mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",files.str(),"ADD_KEY",addKey,mCurrentDB);
  88         mTreeHandler->EndTransaction();
  89     GimmickDebugMessage(3,mProgress<<std::endl);
  90   }
  91
  92   //=====================================================================
  93   void TreeHandlerImageAdder::AddFile( const std::string& filename )
  94   {
  95     GimmickDebugMessage(4,"Adding '"<<filename<<"'"<<std::endl);
  96     std::map< std::string, std::string>  attr;
  97     mTreeHandler->GetTree().GetDescriptor().BuildAttributeMap(attr);
  98
  99     mReader.ReadAttributes(filename,attr);
 100
 101     int lev = mTreeHandler->AddBranch(attr);
 102
 103     // update the progress according to lev
 104     if (lev<mTreeHandler->GetTree().GetNumberOfLevels())
 105       mProgress.IncNumberAddedFiles();
 106   }
 107   //=====================================================================
 108
 109   void TreeHandlerImageAdder::RemoveFile( tree::Node* node)
 110   {
 111                 int n=node->GetNumberOfChildren();
 112                 if(n>0)
 113                 {
 114                         RemoveFiles(node->GetChildrenList());
 115                 }
 116                 else
 117                 {
 118                   std::string path=node->GetAttribute("FullFileName");
 119                   //Gets the add key
 120                   std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
 121                   //Gets the number of files added
 122                   int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
 123                   files=files-1;
 124                   std::stringstream out;
 125                   out<<files;
 126                   //Sets the new number of files
 127                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",addKey,mCurrentDB);
 128                   //Sets the file as removed
 129                   mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",addKey,mCurrentDB);
 130                 }
 131   }
 132
 133   //=====================================================================
 134
 135   void TreeHandlerImageAdder::RemoveFiles(const std::vector<tree::Node*>& nodes)
 136   {
 137           std::vector<tree::Node*>::const_iterator it;
 138           for(it=nodes.begin();it!=nodes.end();++it)
 139           {
 140                 int n=(*it)->GetNumberOfChildren();
 141                 if(n>0)
 142                 {
 143                         RemoveFiles((*it)->GetChildrenList());
 144                 }
 145                 else
 146                 {
 147                    std::string path=(*it)->GetAttribute("FullFileName");
 148                   //Gets the add key
 149                   std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
 150                   //Gets the number of files added
 151                   int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
 152                   files=files-1;
 153                   std::stringstream out;
 154                   out<<files;
 155                   //Sets the new number of files
 156                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",addKey,mCurrentDB);
 157                   //Sets the file as removed
 158                   mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",addKey,mCurrentDB);
 159                 }
 160
 161           }
 162   }
 163
 164   //=====================================================================
 165   void TreeHandlerImageAdder::AddDirectoryRecursor(const std::string &dirpath,
 166                                                    bool recursive,
 167                                                    const std::string &addKey)
 168   {
 169     GimmickDebugMessage(4,"Scanning '"<<dirpath<<"'"<<std::endl);
 170     mProgress.IncNumberScannedDirs();
 171
 172     if ( !fs::exists( dirpath ) ) return;
 173         time_t lastModif=fs::last_write_time(dirpath);
 174
 175     fs::directory_iterator end_itr; // default construction yields past-the-end
 176     for ( fs::directory_iterator itr( dirpath );
 177           itr != end_itr;
 178           ++itr )
 179       {
 180         // If is directory & recurse : do recurse
 181         if ( fs::is_directory(itr->status()) )
 182           {
 183             if (recursive)
 184                 {
 185                         AddDirectoryRecursor( itr->string(), recursive, addKey);
 186                 }
 187           }
 188         else
 189           {
 190                 std::string parent_id;
 191                 bool valid=mTimestampHandler->AddDirectory(dirpath, itr->string(), lastModif, time(0),mCurrentDB);
 192                 if(valid)
 193                 {
 194                         mProgress.IncNumberScannedFiles();
 195                         if (IsHandledFile(itr->string()))
 196                         {
 197                         mProgress.IncNumberHandledFiles();
 198                         AddFile( itr->string() );
 199                         mTreeHandler->GetTopLevelNodeId("FullFileName",itr->string(),parent_id);
 200                         mTimestampHandler->SetAttribute("TopLevelNodeId",parent_id,"PATH",itr->string());
 201                         std::stringstream removedOn;
 202                         removedOn<<time(0);
 203                         mSynchronizer->InsertIgnoreFile(addKey, itr->string(),"0",removedOn.str(),mCurrentDB);
 204                         }
 205
 206                         mProgressSignal(mProgress);
 207                         if (mProgress.GetStop())
 208                         {
 209                         //itr = end_itr;
 210                         break;
 211                         }
 212
 213                 }
 214                 mTimestampHandler->SetAttribute("TopLevelNodeId",parent_id,"PATH",dirpath);
 215           }
 216       }
 217
 218   }
 219   //=======================================================================
 220
 221   //=======================================================================
 222
 223   void TreeHandlerImageAdder::CheckSyncDirectory(const std::string &dirpath,
 224                                                                                                 bool recursive,
 225                                                                                                 bool repair,
 226                                                                                                 bool checkAttributes,
 227                                                                                                 std::vector<std::string> &i_ignorefiles,
 228                                                                                                 std::vector<std::string> & attsModified,
 229                                                                                                 std::vector<std::string> & newfiles)
 230   {
 231     if ( !fs::exists( dirpath ) ) return;
 232     fs::directory_iterator end_itr; // default construction yields past-the-end
 233
 234     for ( fs::directory_iterator itr( dirpath ); itr != end_itr; ++itr )
 235         {
 236         // If is directory & recurse : do recurse
 237         if ( fs::is_directory(itr->status()) )
 238         {
 239             if (recursive)
 240                         {
 241                 CheckSyncDirectory( itr->string(), recursive, repair, checkAttributes, i_ignorefiles, attsModified, newfiles);
 242                         }
 243         }
 244         else
 245         {
 246             if (IsHandledFile(itr->string()))
 247             {
 248                 bool bfound = false;
 249                 for(std::vector<std::string>::iterator it_new = i_ignorefiles.begin(); it_new < i_ignorefiles.end(); ++it_new)
 250                 {
 251                                         if((*it_new) == itr->string())
 252                     {
 253                         bfound = true;
 254                                                 //Additional checking of attributes
 255                                                 if(checkAttributes)
 256                                                 {
 257                                                         CheckAttributes(repair,(*it_new),attsModified);
 258                                                 }
 259                         i_ignorefiles.erase(it_new);
 260                         break;
 261                     }
 262                                 }
 263                                 if(!bfound && i_ignorefiles.size()>0 )
 264                 {
 265                     newfiles.push_back( itr->string() );
 266                 }
 267                         }
 268                 }
 269          }
 270   }
 271
 272   //=======================================================================
 273
 274   //=======================================================================
 275
 276   std::string TreeHandlerImageAdder::Synchronize(bool repair, bool checkAttributes)
 277   {
 278           std::vector<AddList> fileList;
 279           std::vector<std::string> ignoreList;
 280           std::vector<std::string> newFiles;
 281           std::vector<std::string> attsModified;
 282           std::stringstream mess;
 283           std::vector<AddList>::iterator iter;
 284
 285           //Gets the list of added files
 286           mSynchronizer->GetFileList(fileList,mCurrentDB);
 287
 288           std::vector<std::string>::iterator i;
 289           //Actions to take if the user doesn't want to repair
 290           if(!repair)
 291           {
 292                 //Iterates to see if they are in sync
 293                 for(iter=fileList.begin();iter!=fileList.end();++iter)
 294                 {
 295                         mSynchronizer->GetIgnoredFiles((*iter).key,ignoreList);
 296                         bool rec=true;
 297                         if((*iter).recursive=="0"){rec=false;}
 298                         CheckSyncDirectory((*iter).path,rec,repair,checkAttributes,ignoreList,attsModified,newFiles);
 299                 }
 300
 301                 //Add to message the result of new found files
 302                 mess<<"New Files Found: "<<newFiles.size()<<std::endl;
 303                 if(newFiles.size()>0)
 304                 {
 305                         mess<<"Filenames: "<<std::endl;
 306                         for(i=newFiles.begin();i!=newFiles.end();++i)
 307                         {
 308                                 mess<<*i<<std::endl;
 309                         }
 310                 }
 311
 312                 //Add to message the result of missing files
 313                 mess<<"Missing Files: "<<ignoreList.size()<<std::endl;
 314                 if(ignoreList.size()>0)
 315                 {
 316                         mess<<"Filenames: "<<std::endl;
 317                         for(i=ignoreList.begin();i!=ignoreList.end();++i)
 318                         {
 319                                 mess<<*i<<std::endl;
 320                         }
 321                 }
 322
 323                 //In the case that the user wants to check the attributes...
 324                 if(checkAttributes)
 325                 {
 326                         //... add to message the result of files that have been changed.
 327                         mess<<"Files with different attributes: "<<attsModified.size()<<std::endl;
 328                         if(attsModified.size()>0)
 329                         {
 330                                 mess<<"Filenames: "<<std::endl;
 331                                 for(i=attsModified.begin();i!=attsModified.end();++i)
 332                                 {
 333                                         mess<<*i<<std::endl;
 334                                 }
 335                         }
 336                 }
 337
 338           }
 339
 340           //Actions to take if the user wants to repair
 341           else
 342           {
 343                   int nf=0;
 344                 //Iterates to see if they are in sync
 345                 for(iter=fileList.begin();iter!=fileList.end();++iter)
 346                 {
 347                         mSynchronizer->GetIgnoredFiles((*iter).key,ignoreList);
 348                         bool rec=true;
 349                         if((*iter).recursive=="0"){rec=false;}
 350                         CheckSyncDirectory((*iter).path,rec,repair,checkAttributes,ignoreList,attsModified,newFiles);
 351
 352                         //For the new files, add them
 353                         for (i=newFiles.begin();i!=newFiles.end();++i)
 354                         {
 355                         mTimestampHandler->AddFile((*i), fs::last_write_time(*i), time(0),mCurrentDB);
 356                         if (IsHandledFile(*i))
 357                         {
 358                                 std::stringstream removedOn;
 359                                 removedOn<<time(0);
 360                                 mSynchronizer->InsertIgnoreFile((*iter).key,(*i),"0",removedOn.str(),mCurrentDB);
 361                                 //Gets the number of files added
 362                                 int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",(*iter).key,mCurrentDB)).c_str());
 363                                 files=files+1;
 364                                 std::stringstream out;
 365                                 out<<files;
 366                                 //Sets the new number of files
 367                                 mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",(*iter).key,mCurrentDB);
 368                                 AddFile(*i);
 369                         }
 370                         }
 371                         nf+=newFiles.size();
 372                         newFiles.clear();
 373
 374                 }
 375                 //Reports number of added files
 376                 mess<<"Files Added: "<<nf<<std::endl;
 377
 378                 //Removes the necessary files and reports the results
 379                 if(ignoreList.size()>0)
 380                 {
 381                         tree::Node* node;
 382                         mTreeHandler->LoadChildren(NULL,4);
 383                         for(i=ignoreList.begin();i!=ignoreList.end();++i)
 384                         {
 385                                 FindNode(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",*i,node);
 386                                 RemoveFile(node);
 387                                 mTreeHandler->Remove(node);
 388                         }
 389                 }
 390                 mess<<"Files Removed: "<<ignoreList.size()<<std::endl;
 391                 //In the case that the user wants to check the attributes...
 392                 if(checkAttributes)
 393                 {
 394                         //... add to message the result of files that were modified.
 395                         mess<<"Files Modified: "<<attsModified.size()<<std::endl;
 396                 }
 397           }
 398           return mess.str();
 399
 400   }
 401   //=======================================================================
 402
 403   void TreeHandlerImageAdder::CheckAttributes(bool repair, std::string& file, std::vector<std::string>& attsModified)
 404   {
 405           std::map< std::string, std::string>  attr;
 406           mTreeHandler->GetTree().GetDescriptor().BuildAttributeMap(attr);
 407       mReader.ReadAttributes(file,attr);
 408           tree::LevelDescriptor::AttributeDescriptorListType adl= mTreeHandler->GetTree().GetAttributeDescriptorList(mTreeHandler->GetTree().GetNumberOfLevels()-1);
 409           tree::LevelDescriptor::AttributeDescriptorListType::const_iterator a;
 410           for (a = adl.begin();a!=adl.end();++a)
 411           {
 412            std::string databaseVal;
 413            mTreeHandler->GetAttribute("Image","FullFileName",file,a->GetKey(),databaseVal);
 414            std::string fileVal=attr.find(a->GetKey())->second;
 415            if ( a->GetFlags()==0 && databaseVal.compare(fileVal)!=0 )
 416             {
 417                         if(repair)
 418                         {
 419                                 mTreeHandler->SetAttribute("Image",a->GetKey(),fileVal,"FullFileName", file);
 420                         }
 421                         attsModified.push_back(file);
 422             }
 423
 424           }
 425
 426   }
 427
 428   //=======================================================================
 429
 430
 431   void TreeHandlerImageAdder::FindNode(tree::Node* parent, int level, const std::string& searchParam, const std::string& searchVal, tree::Node*& node)
 432   {
 433           if(level>1)
 434           {
 435                   std::vector<tree::Node*>::iterator iter;
 436                   for(iter=parent->GetChildrenList().begin();iter!=parent->GetChildrenList().end();++iter)
 437                   {
 438                           FindNode(*iter,level-1,searchParam,searchVal,node);
 439                   }
 440           }
 441           else
 442           {
 443                   if(parent->GetAttribute(searchParam).compare(searchVal)==0)
 444                   {
 445                           node=parent;
 446                   }
 447
 448           }
 449   }
 450
 451   //=======================================================================
 452   void TreeHandlerImageAdder::FindNodePartial(tree::Node* parent, int level, const std::string& searchParam, const std::string& searchVal, tree::Node*& node)
 453   {
 454           if(level>1)
 455           {
 456                   std::vector<tree::Node*>::iterator iter;
 457                   for(iter=parent->GetChildrenList().begin();iter!=parent->GetChildrenList().end() && node==0 ;++iter)
 458                   {
 459                           FindNodePartial(*iter,level-1,searchParam,searchVal,node);
 460                   }
 461           }
 462           else
 463           {
 464                   if(parent->GetAttribute(searchParam).find(searchVal)<9000)
 465                   {
 466                           node=parent;
 467                           return;
 468                   }
 469
 470           }
 471   }
 472
 473   //=======================================================================
 474
 475   void TreeHandlerImageAdder::CopyFiles(const std::vector<std::string>& filenames, const std::string directory  )
 476   {
 477           std::vector<std::string>::const_iterator i;
 478           if(!boost::filesystem::exists(directory))
 479           {
 480           boost::filesystem::create_directory(boost::filesystem::path(directory));
 481           mTimestampHandler->AddDirectory("",directory,fs::last_write_time(directory)+10, time(0),mCurrentDB);
 482           mSynchronizer->InsertAddOp(directory,"0","0",mCurrentDB);
 483           }
 484           std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",directory,mCurrentDB);
 485           std::string parent_id=mTimestampHandler->IsIndexed(directory,mCurrentDB);
 486           size_t last;
 487           std::vector<std::string> newNames;
 488           for(i=filenames.begin();i!=filenames.end();++i)
 489           {
 490                   std::string dir=directory.c_str();
 491                   mTimestampHandler->CleanPath(dir);
 492                   if(boost::filesystem::exists(*i) && (*i).find(dir)==std::string::npos)
 493                   {
 494                   std::string path=*i;
 495                   last=(*i).find_last_of('/');
 496                   std::string f="\\"+(*i).substr(last+1);
 497
 498                   int p=1;
 499                   std::stringstream out;
 500                   out<<directory<<f;
 501                   while(boost::filesystem::exists(out.str()))
 502                   {
 503                           out.str("");
 504                           out<<directory<<f.substr(0,f.size()-4)<<"("<<p<<")"<<f.substr(f.size()-4);
 505                           p++;
 506                   }
 507                   std::string result=out.str();
 508                   mTimestampHandler->CleanPath(result);
 509                   boost::filesystem::copy_file((*i),result);
 510
 511                   //To update image database
 512                   mTreeHandler->SetAttribute("Image","FullFileName",result,"FullFileName", (*i));
 513
 514                   //To update timestamp database
 515                   mTimestampHandler->SetAttribute("PATH",result,"PATH",(*i));
 516                   mTimestampHandler->SetAttribute("PARENT_ID",parent_id,"PATH",result);
 517                   std::stringstream t;
 518                   t<<fs::last_write_time(directory)+10;
 519                   mTimestampHandler->SetAttribute("LastModified",t.str(),"PATH",result);
 520
 521                   //To update maintenance database
 522                   //1.Add the new path and increase number of children on new operation.
 523                   std::stringstream removedOn;
 524                   removedOn<<time(0);
 525                   //Inserts the file
 526                   mSynchronizer->InsertIgnoreFile(addKey, result,"0",removedOn.str(),mCurrentDB);
 527                   //Gets the number of files added
 528                   int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
 529                   files=files+1;
 530                   std::stringstream fil;
 531                   fil<<files;
 532                   //Sets the new number of files
 533                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",fil.str(),"ADD_KEY",addKey,mCurrentDB);
 534                   fil.str("");
 535
 536                   //2.Set the old path as removed and decrease number of children on old operation.
 537                   //Gets the old add key
 538                   std::string oldAddKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
 539                   //Sets the file as removed
 540                   mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",oldAddKey,mCurrentDB);
 541                   //Gets the number of files added
 542                   files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",oldAddKey,mCurrentDB)).c_str());
 543                   files=files-1;
 544                   fil<<files;
 545                   //Sets the new number of files
 546                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",fil.str(),"ADD_KEY",oldAddKey,mCurrentDB);
 547
 548                   }
 549
 550           }
 551   }
 552
 553   //=======================================================================
 554
 555   void TreeHandlerImageAdder::DeleteDriveFromMainDB(const std::string& drive)
 556   {
 557           //Delete from local database and others
 558           tree::Node* node=0;
 559           mTreeHandler->LoadChildren(NULL,4);
 560           FindNodePartial(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",drive,node);
 561           while(node!=0)
 562           {
 563           mTreeHandler->Remove(node);
 564           node=0;
 565           mTreeHandler->LoadChildren(NULL,4);
 566           FindNodePartial(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",drive,node);
 567           }
 568   }
 569
 570    //=======================================================================
 571
 572   void TreeHandlerImageAdder::DeleteDriveFromOtherDB(const std::string& drive)
 573   {
 574           //Delete from timestamp
 575           mTimestampHandler->RemoveEntries("FILES", "PATH", "LIKE", drive+"%");
 576
 577           //Delete from maintenance
 578           mSynchronizer->RemoveEntries("ADD_OPS", "PATH", "LIKE", drive+"%");
 579           mSynchronizer->RemoveEntries("IGNORED_FILES", "PATH", "LIKE", drive+"%");
 580   }
 581
 582   //=======================================================================
 583   void TreeHandlerImageAdder::EditField(tree::Node* node, const std::string& name, const std::string& key, const std::string& val)
 584   {
 585           node->SetAttribute(key,val);
 586           mTreeHandler->SetAttribute(node,key,val);
 587   }
 588
 589
 590 }