src2/creaImageIOTreeHandlerImageAdder.cpp

   1 #include <creaImageIOTreeHandlerImageAdder.h>
   2 #include <creaImageIOSystem.h>
   3 #include "boost/filesystem.hpp"
   4 #include <boost/filesystem/operations.hpp>
   5 #include <boost/utility.hpp>
   6
   7
   8 namespace fs = boost::filesystem;
   9 using boost::filesystem::path;
  10 using boost::next;
  11 using boost::prior;
  12
  13
  14 using namespace crea;
  15
  16 namespace creaImageIO
  17 {
  18   //====================================================================
  19   // Ctor
  20   TreeHandlerImageAdder::TreeHandlerImageAdder(TreeHandler* tree)
  21     : mTreeHandler(tree)
  22   {
  23   }
  24   // Dtor
  25   TreeHandlerImageAdder::~TreeHandlerImageAdder()
  26   {
  27   }
  28   //====================================================================
  29
  30   //====================================================================
  31   void TreeHandlerImageAdder::ConnectProgressObserver(ProgressCallbackType callback)
  32   {
  33     mProgressSignal.connect(callback);
  34   }
  35   //====================================================================
  36
  37   //=====================================================================
  38   bool TreeHandlerImageAdder::IsHandledFile( const std::string& filename)
  39   {
  40     return (mReader.CanRead(filename));
  41   }
  42   //=====================================================================
  43
  44   //=====================================================================
  45   void TreeHandlerImageAdder::AddFiles( const std::vector<std::string>& filenames)
  46   {
  47     mProgress.Reset();
  48
  49     unsigned int nbf = filenames.size();
  50     std::vector<std::string>::const_iterator i;
  51     for (i=filenames.begin();i!=filenames.end();++i)
  52       {
  53         mTimestampHandler->AddFile((*i), fs::last_write_time(*i), time(0),mCurrentDB);
  54         mProgress.IncNumberScannedFiles();
  55         if (IsHandledFile(*i))
  56           {
  57             mProgress.IncNumberHandledFiles();
  58                 mSynchronizer->InsertAddOp((*i),"0","1",mCurrentDB);
  59                 std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",(*i),mCurrentDB);
  60                 std::stringstream removedOn;
  61                 removedOn<<time(0);
  62                 mSynchronizer->InsertIgnoreFile(addKey,(*i),"0",removedOn.str(),mCurrentDB);
  63             AddFile(*i);
  64           }
  65         mProgressSignal(mProgress);
  66         if (mProgress.GetStop()) break;
  67       }
  68   }
  69   //=====================================================================
  70
  71   //=====================================================================
  72   void TreeHandlerImageAdder::AddDirectory( const std::string& directory,
  73                                             bool recurse)
  74   {
  75     mProgress.Reset();
  76         std::stringstream files;
  77
  78         std::stringstream rec;
  79         rec<<recurse;
  80         mSynchronizer->InsertAddOp(directory,rec.str(),"0",mCurrentDB);
  81         std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",directory,mCurrentDB);
  82         AddDirectoryRecursor( directory, recurse, addKey );
  83
  84         int nFiles=GetProgress().GetNumberAddedFiles();
  85         files<<nFiles;
  86         mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",files.str(),"ADD_KEY",addKey,mCurrentDB);
  87     GimmickDebugMessage(3,mProgress<<std::endl);
  88   }
  89
  90   //=====================================================================
  91   void TreeHandlerImageAdder::AddFile( const std::string& filename )
  92   {
  93     GimmickDebugMessage(4,"Adding '"<<filename<<"'"<<std::endl);
  94     std::map< std::string, std::string>  attr;
  95     mTreeHandler->GetTree().GetDescriptor().BuildAttributeMap(attr);
  96
  97     mReader.ReadAttributes(filename,attr);
  98
  99     int lev = mTreeHandler->AddBranch(attr);
 100
 101     // update the progress according to lev
 102     if (lev<mTreeHandler->GetTree().GetNumberOfLevels())
 103       mProgress.IncNumberAddedFiles();
 104   }
 105   //=====================================================================
 106
 107   void TreeHandlerImageAdder::RemoveFile( tree::Node* node)
 108   {
 109                 int n=node->GetNumberOfChildren();
 110                 if(n>0)
 111                 {
 112                         RemoveFiles(node->GetChildrenList());
 113                 }
 114                 else
 115                 {
 116                   std::string path=node->GetAttribute("FullFileName");
 117                   //Gets the add key
 118                   std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
 119                   //Gets the number of files added
 120                   int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
 121                   files=files-1;
 122                   std::stringstream out;
 123                   out<<files;
 124                   //Sets the new number of files
 125                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",addKey,mCurrentDB);
 126                   //Sets the file as removed
 127                   mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",addKey,mCurrentDB);
 128                 }
 129   }
 130
 131   //=====================================================================
 132
 133   void TreeHandlerImageAdder::RemoveFiles(const std::vector<tree::Node*>& nodes)
 134   {
 135           std::vector<tree::Node*>::const_iterator it;
 136           for(it=nodes.begin();it!=nodes.end();++it)
 137           {
 138                 int n=(*it)->GetNumberOfChildren();
 139                 if(n>0)
 140                 {
 141                         RemoveFiles((*it)->GetChildrenList());
 142                 }
 143                 else
 144                 {
 145                    std::string path=(*it)->GetAttribute("FullFileName");
 146                   //Gets the add key
 147                   std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
 148                   //Gets the number of files added
 149                   int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
 150                   files=files-1;
 151                   std::stringstream out;
 152                   out<<files;
 153                   //Sets the new number of files
 154                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",addKey,mCurrentDB);
 155                   //Sets the file as removed
 156                   mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",addKey,mCurrentDB);
 157                 }
 158
 159           }
 160   }
 161
 162   //=====================================================================
 163   void TreeHandlerImageAdder::AddDirectoryRecursor(const std::string &dirpath,
 164                                                    bool recursive,
 165                                                    const std::string &addKey)
 166   {
 167     GimmickDebugMessage(4,"Scanning '"<<dirpath<<"'"<<std::endl);
 168     mProgress.IncNumberScannedDirs();
 169
 170     if ( !fs::exists( dirpath ) ) return;
 171         time_t lastModif=fs::last_write_time(dirpath);
 172
 173     fs::directory_iterator end_itr; // default construction yields past-the-end
 174     for ( fs::directory_iterator itr( dirpath );
 175           itr != end_itr;
 176           ++itr )
 177       {
 178         // If is directory & recurse : do recurse
 179         if ( fs::is_directory(itr->status()) )
 180           {
 181             if (recursive)
 182                 {
 183                         AddDirectoryRecursor( itr->string(), recursive, addKey);
 184                 }
 185           }
 186         else
 187           {
 188                 std::string parent_id;
 189                 bool valid=mTimestampHandler->AddDirectory(dirpath, itr->string(), lastModif, time(0),mCurrentDB);
 190                 if(valid)
 191                 {
 192                         mProgress.IncNumberScannedFiles();
 193                         if (IsHandledFile(itr->string()))
 194                         {
 195                         mProgress.IncNumberHandledFiles();
 196                         AddFile( itr->string() );
 197                         mTreeHandler->GetTopLevelNodeId("FullFileName",itr->string(),parent_id);
 198                         mTimestampHandler->SetAttribute("TopLevelNodeId",parent_id,"PATH",itr->string());
 199                         std::stringstream removedOn;
 200                         removedOn<<time(0);
 201                         mSynchronizer->InsertIgnoreFile(addKey, itr->string(),"0",removedOn.str(),mCurrentDB);
 202                         }
 203
 204                         mProgressSignal(mProgress);
 205                         if (mProgress.GetStop())
 206                         {
 207                         //itr = end_itr;
 208                         break;
 209                         }
 210
 211                 }
 212                 mTimestampHandler->SetAttribute("TopLevelNodeId",parent_id,"PATH",dirpath);
 213           }
 214       }
 215
 216   }
 217   //=======================================================================
 218
 219   //=======================================================================
 220
 221   void TreeHandlerImageAdder::CheckSyncDirectory(const std::string &dirpath,
 222                                                                                                 bool recursive,
 223                                                                                                 bool repair,
 224                                                                                                 bool checkAttributes,
 225                                                                                                 std::vector<std::string> &i_ignorefiles,
 226                                                                                                 std::vector<std::string> & attsModified,
 227                                                                                                 std::vector<std::string> & newfiles)
 228   {
 229     if ( !fs::exists( dirpath ) ) return;
 230     fs::directory_iterator end_itr; // default construction yields past-the-end
 231
 232     for ( fs::directory_iterator itr( dirpath ); itr != end_itr; ++itr )
 233         {
 234         // If is directory & recurse : do recurse
 235         if ( fs::is_directory(itr->status()) )
 236         {
 237             if (recursive)
 238                         {
 239                 CheckSyncDirectory( itr->string(), recursive, repair, checkAttributes, i_ignorefiles, attsModified, newfiles);
 240                         }
 241         }
 242         else
 243         {
 244             if (IsHandledFile(itr->string()))
 245             {
 246                 bool bfound = false;
 247                 for(std::vector<std::string>::iterator it_new = i_ignorefiles.begin(); it_new < i_ignorefiles.end(); ++it_new)
 248                 {
 249                                         if((*it_new) == itr->string())
 250                     {
 251                         bfound = true;
 252                                                 //Additional checking of attributes
 253                                                 if(checkAttributes)
 254                                                 {
 255                                                         CheckAttributes(repair,(*it_new),attsModified);
 256                                                 }
 257                         i_ignorefiles.erase(it_new);
 258                         break;
 259                     }
 260                                 }
 261                                 if(!bfound && i_ignorefiles.size()>0 )
 262                 {
 263                     newfiles.push_back( itr->string() );
 264                 }
 265                         }
 266                 }
 267          }
 268   }
 269
 270   //=======================================================================
 271
 272   //=======================================================================
 273
 274   std::string TreeHandlerImageAdder::Synchronize(bool repair, bool checkAttributes)
 275   {
 276           std::vector<AddList> fileList;
 277           std::vector<std::string> ignoreList;
 278           std::vector<std::string> newFiles;
 279           std::vector<std::string> attsModified;
 280           std::stringstream mess;
 281           std::vector<AddList>::iterator iter;
 282
 283           //Gets the list of added files
 284           mSynchronizer->GetFileList(fileList,mCurrentDB);
 285
 286           std::vector<std::string>::iterator i;
 287           //Actions to take if the user doesn't want to repair
 288           if(!repair)
 289           {
 290                 //Iterates to see if they are in sync
 291                 for(iter=fileList.begin();iter!=fileList.end();++iter)
 292                 {
 293                         mSynchronizer->GetIgnoredFiles((*iter).key,ignoreList);
 294                         bool rec=true;
 295                         if((*iter).recursive=="0"){rec=false;}
 296                         CheckSyncDirectory((*iter).path,rec,repair,checkAttributes,ignoreList,attsModified,newFiles);
 297                 }
 298
 299                 //Add to message the result of new found files
 300                 mess<<"New Files Found: "<<newFiles.size()<<std::endl;
 301                 if(newFiles.size()>0)
 302                 {
 303                         mess<<"Filenames: "<<std::endl;
 304                         for(i=newFiles.begin();i!=newFiles.end();++i)
 305                         {
 306                                 mess<<*i<<std::endl;
 307                         }
 308                 }
 309
 310                 //Add to message the result of missing files
 311                 mess<<"Missing Files: "<<ignoreList.size()<<std::endl;
 312                 if(ignoreList.size()>0)
 313                 {
 314                         mess<<"Filenames: "<<std::endl;
 315                         for(i=ignoreList.begin();i!=ignoreList.end();++i)
 316                         {
 317                                 mess<<*i<<std::endl;
 318                         }
 319                 }
 320
 321                 //In the case that the user wants to check the attributes...
 322                 if(checkAttributes)
 323                 {
 324                         //... add to message the result of files that have been changed.
 325                         mess<<"Files with different attributes: "<<attsModified.size()<<std::endl;
 326                         if(attsModified.size()>0)
 327                         {
 328                                 mess<<"Filenames: "<<std::endl;
 329                                 for(i=attsModified.begin();i!=attsModified.end();++i)
 330                                 {
 331                                         mess<<*i<<std::endl;
 332                                 }
 333                         }
 334                 }
 335
 336           }
 337
 338           //Actions to take if the user wants to repair
 339           else
 340           {
 341                   int nf=0;
 342                 //Iterates to see if they are in sync
 343                 for(iter=fileList.begin();iter!=fileList.end();++iter)
 344                 {
 345                         mSynchronizer->GetIgnoredFiles((*iter).key,ignoreList);
 346                         bool rec=true;
 347                         if((*iter).recursive=="0"){rec=false;}
 348                         CheckSyncDirectory((*iter).path,rec,repair,checkAttributes,ignoreList,attsModified,newFiles);
 349
 350                         //For the new files, add them
 351                         for (i=newFiles.begin();i!=newFiles.end();++i)
 352                         {
 353                         mTimestampHandler->AddFile((*i), fs::last_write_time(*i), time(0),mCurrentDB);
 354                         if (IsHandledFile(*i))
 355                         {
 356                                 std::stringstream removedOn;
 357                                 removedOn<<time(0);
 358                                 mSynchronizer->InsertIgnoreFile((*iter).key,(*i),"0",removedOn.str(),mCurrentDB);
 359                                 //Gets the number of files added
 360                                 int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",(*iter).key,mCurrentDB)).c_str());
 361                                 files=files+1;
 362                                 std::stringstream out;
 363                                 out<<files;
 364                                 //Sets the new number of files
 365                                 mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",out.str(),"ADD_KEY",(*iter).key,mCurrentDB);
 366                                 AddFile(*i);
 367                         }
 368                         }
 369                         nf+=newFiles.size();
 370                         newFiles.clear();
 371
 372                 }
 373                 //Reports number of added files
 374                 mess<<"Files Added: "<<nf<<std::endl;
 375
 376                 //Removes the necessary files and reports the results
 377                 if(ignoreList.size()>0)
 378                 {
 379                         tree::Node* node;
 380                         mTreeHandler->LoadChildren(NULL,4);
 381                         for(i=ignoreList.begin();i!=ignoreList.end();++i)
 382                         {
 383                                 FindNode(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",*i,node);
 384                                 RemoveFile(node);
 385                                 mTreeHandler->Remove(node);
 386                         }
 387                 }
 388                 mess<<"Files Removed: "<<ignoreList.size()<<std::endl;
 389                 //In the case that the user wants to check the attributes...
 390                 if(checkAttributes)
 391                 {
 392                         //... add to message the result of files that were modified.
 393                         mess<<"Files Modified: "<<attsModified.size()<<std::endl;
 394                 }
 395           }
 396           return mess.str();
 397
 398   }
 399   //=======================================================================
 400
 401   void TreeHandlerImageAdder::CheckAttributes(bool repair, std::string& file, std::vector<std::string>& attsModified)
 402   {
 403           std::map< std::string, std::string>  attr;
 404           mTreeHandler->GetTree().GetDescriptor().BuildAttributeMap(attr);
 405       mReader.ReadAttributes(file,attr);
 406           tree::LevelDescriptor::AttributeDescriptorListType adl= mTreeHandler->GetTree().GetAttributeDescriptorList(mTreeHandler->GetTree().GetNumberOfLevels()-1);
 407           tree::LevelDescriptor::AttributeDescriptorListType::const_iterator a;
 408           for (a = adl.begin();a!=adl.end();++a)
 409           {
 410            std::string databaseVal;
 411            mTreeHandler->GetAttribute("Image","FullFileName",file,a->GetKey(),databaseVal);
 412            std::string fileVal=attr.find(a->GetKey())->second;
 413            if ( a->GetFlags()==0 && databaseVal.compare(fileVal)!=0 )
 414             {
 415                         if(repair)
 416                         {
 417                                 mTreeHandler->SetAttribute("Image",a->GetKey(),fileVal,"FullFileName", file);
 418                         }
 419                         attsModified.push_back(file);
 420             }
 421
 422           }
 423
 424   }
 425
 426   //=======================================================================
 427
 428
 429   void TreeHandlerImageAdder::FindNode(tree::Node* parent, int level, const std::string& searchParam, const std::string& searchVal, tree::Node*& node)
 430   {
 431           if(level>1)
 432           {
 433                   std::vector<tree::Node*>::iterator iter;
 434                   for(iter=parent->GetChildrenList().begin();iter!=parent->GetChildrenList().end();++iter)
 435                   {
 436                           FindNode(*iter,level-1,searchParam,searchVal,node);
 437                   }
 438           }
 439           else
 440           {
 441                   if(parent->GetAttribute(searchParam).compare(searchVal)==0)
 442                   {
 443                           node=parent;
 444                   }
 445
 446           }
 447   }
 448
 449   //=======================================================================
 450   void TreeHandlerImageAdder::FindNodePartial(tree::Node* parent, int level, const std::string& searchParam, const std::string& searchVal, tree::Node*& node)
 451   {
 452           if(level>1)
 453           {
 454                   std::vector<tree::Node*>::iterator iter;
 455                   for(iter=parent->GetChildrenList().begin();iter!=parent->GetChildrenList().end() && node==0 ;++iter)
 456                   {
 457                           FindNodePartial(*iter,level-1,searchParam,searchVal,node);
 458                   }
 459           }
 460           else
 461           {
 462                   if(parent->GetAttribute(searchParam).find(searchVal)<9000)
 463                   {
 464                           node=parent;
 465                           return;
 466                   }
 467
 468           }
 469   }
 470
 471   //=======================================================================
 472
 473   void TreeHandlerImageAdder::CopyFiles(const std::vector<std::string>& filenames, const std::string directory  )
 474   {
 475           std::vector<std::string>::const_iterator i;
 476           if(!boost::filesystem::exists(directory))
 477           {
 478           boost::filesystem::create_directory(boost::filesystem::path(directory));
 479           mTimestampHandler->AddDirectory("",directory,fs::last_write_time(directory)+10, time(0),mCurrentDB);
 480           mSynchronizer->InsertAddOp(directory,"0","0",mCurrentDB);
 481           }
 482           std::string addKey=mSynchronizer->GetAttribute("ADD_KEY","ADD_OPS","PATH",directory,mCurrentDB);
 483           std::string parent_id=mTimestampHandler->IsIndexed(directory,mCurrentDB);
 484           size_t last;
 485           std::vector<std::string> newNames;
 486           for(i=filenames.begin();i!=filenames.end();++i)
 487           {
 488                   std::string dir=directory.c_str();
 489                   mTimestampHandler->CleanPath(dir);
 490                   if(boost::filesystem::exists(*i) && (*i).find(dir)==std::string::npos)
 491                   {
 492                   std::string path=*i;
 493                   last=(*i).find_last_of('/');
 494                   std::string f="\\"+(*i).substr(last+1);
 495
 496                   int p=1;
 497                   std::stringstream out;
 498                   out<<directory<<f;
 499                   while(boost::filesystem::exists(out.str()))
 500                   {
 501                           out.str("");
 502                           out<<directory<<f.substr(0,f.size()-4)<<"("<<p<<")"<<f.substr(f.size()-4);
 503                           p++;
 504                   }
 505                   std::string result=out.str();
 506                   mTimestampHandler->CleanPath(result);
 507                   boost::filesystem::copy_file((*i),result);
 508
 509                   //To update image database
 510                   mTreeHandler->SetAttribute("Image","FullFileName",result,"FullFileName", (*i));
 511
 512                   //To update timestamp database
 513                   mTimestampHandler->SetAttribute("PATH",result,"PATH",(*i));
 514                   mTimestampHandler->SetAttribute("PARENT_ID",parent_id,"PATH",result);
 515                   std::stringstream t;
 516                   t<<fs::last_write_time(directory)+10;
 517                   mTimestampHandler->SetAttribute("LastModified",t.str(),"PATH",result);
 518
 519                   //To update maintenance database
 520                   //1.Add the new path and increase number of children on new operation.
 521                   std::stringstream removedOn;
 522                   removedOn<<time(0);
 523                   //Inserts the file
 524                   mSynchronizer->InsertIgnoreFile(addKey, result,"0",removedOn.str(),mCurrentDB);
 525                   //Gets the number of files added
 526                   int files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",addKey,mCurrentDB)).c_str());
 527                   files=files+1;
 528                   std::stringstream fil;
 529                   fil<<files;
 530                   //Sets the new number of files
 531                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",fil.str(),"ADD_KEY",addKey,mCurrentDB);
 532                   fil.str("");
 533
 534                   //2.Set the old path as removed and decrease number of children on old operation.
 535                   //Gets the old add key
 536                   std::string oldAddKey=mSynchronizer->GetAttribute("ADD_KEY","IGNORED_FILES","PATH",path,mCurrentDB);
 537                   //Sets the file as removed
 538                   mSynchronizer->SetAttribute("REMOVE","IGNORED_FILES","1","PATH = '"+path+"' AND ADD_KEY",oldAddKey,mCurrentDB);
 539                   //Gets the number of files added
 540                   files=atoi((mSynchronizer->GetAttribute("FILES_ADDED","ADD_OPS","ADD_KEY",oldAddKey,mCurrentDB)).c_str());
 541                   files=files-1;
 542                   fil<<files;
 543                   //Sets the new number of files
 544                   mSynchronizer->SetAttribute("FILES_ADDED","ADD_OPS",fil.str(),"ADD_KEY",oldAddKey,mCurrentDB);
 545
 546                   }
 547
 548           }
 549   }
 550
 551   //=======================================================================
 552
 553   void TreeHandlerImageAdder::DeleteDriveFromMainDB(const std::string& drive)
 554   {
 555           //Delete from local database and others
 556           tree::Node* node=0;
 557           mTreeHandler->LoadChildren(NULL,4);
 558           FindNodePartial(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",drive,node);
 559           while(node!=0)
 560           {
 561           mTreeHandler->Remove(node);
 562           node=0;
 563           mTreeHandler->LoadChildren(NULL,4);
 564           FindNodePartial(mTreeHandler->GetTree().GetChildrenList()[0],3,"FullFileName",drive,node);
 565           }
 566   }
 567
 568    //=======================================================================
 569
 570   void TreeHandlerImageAdder::DeleteDriveFromOtherDB(const std::string& drive)
 571   {
 572           //Delete from timestamp
 573           mTimestampHandler->RemoveEntries("FILES", "PATH", "LIKE", drive+"%");
 574
 575           //Delete from maintenance
 576           mSynchronizer->RemoveEntries("ADD_OPS", "PATH", "LIKE", drive+"%");
 577           mSynchronizer->RemoveEntries("IGNORED_FILES", "PATH", "LIKE", drive+"%");
 578   }
 579
 580
 581 }