001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import java.nio.file.Paths;
028import java.nio.file.Files;
029import java.io.IOException;
030import java.nio.file.OpenOption;
031import java.nio.file.StandardOpenOption;
032import java.nio.channels.FileChannel;
033import java.nio.channels.FileLock;
034import java.nio.file.Path;
035import java.nio.ByteBuffer;
036import com.killcoding.log.Logger;
037import java.net.URI;
038import java.util.Comparator;
039import java.util.List;
040import java.util.ArrayList;
041import java.nio.channels.SeekableByteChannel;
042import java.io.ByteArrayOutputStream;
043import java.nio.file.attribute.BasicFileAttributes;
044import java.nio.file.attribute.FileTime;
045import java.time.Instant;
046import java.util.Arrays;
047import java.io.ByteArrayInputStream;
048import java.sql.Timestamp;
049import java.util.stream.Stream;
050import java.util.stream.Collectors;
051import java.util.HashSet;
052import java.util.Set;
053import java.nio.file.DirectoryStream;
054import java.nio.file.CopyOption;
055import java.nio.file.LinkOption;
056import com.killcoding.tool.CommonTools;
057import com.killcoding.file.RemoteFile;
058import com.killcoding.log.LoggerFactory;
059import java.util.concurrent.Future;
060import com.killcoding.tool.ConfigProperties;
061import com.killcoding.datasource.Clock;
062import java.util.concurrent.Callable;
063import com.killcoding.file.DiskFile;
064import java.nio.file.attribute.FileAttribute;
065import java.text.SimpleDateFormat;
066import com.killcoding.cache.CacheArray;
067import com.killcoding.cache.CacheArrayFilter;
068import java.util.Map;
069import java.util.concurrent.ExecutorService;
070import java.util.concurrent.Executors;
071import java.nio.file.NoSuchFileException;
072import java.util.concurrent.ConcurrentHashMap;
073import java.util.Iterator;
074import java.util.Date;
075import java.time.ZonedDateTime;
076import java.time.LocalDateTime;
077import java.time.ZoneId;
078import java.time.format.DateTimeFormatter;
079import java.util.Calendar;
080import java.util.TimeZone;
081import java.text.ParseException;
082import java.util.Collections;
083import java.util.HashMap;
084import java.nio.file.FileAlreadyExistsException;
085import java.util.LinkedList;
086
087public class DiskFile extends BaseFile {
088
089    protected static Integer MAX_POOL_SIZE = 100;
090        private static final Map<String, Long> QUEUE_PATH_MAPPING = new ConcurrentHashMap<String, Long>();
091
092        private static ExecutorService splitPool = null;
093
094    protected boolean copyStructureOnly = false;
095        private FileChannel channel = null;
096        private FileLock lock = null;
097        private long modifyTimeMs = 0L;
098
099        public DiskFile(String path) {
100                super(path);
101        }
102
103        public static synchronized void initPool(int poolSize) {
104                if (splitPool == null) {
105                    MAX_POOL_SIZE = poolSize;
106                        LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
107                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
108                }
109        }
110        
111        private static synchronized void initDefaultPool(){
112                if (splitPool == null) {
113                    LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
114                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
115                }
116        }
117
118        public void split(int partSize, FilePart filePart) throws IOException {
119                if (isLink()) {
120                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
121                }
122                if (isDir()) {
123                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
124                }
125                if (!exists()) {
126                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
127                }
128                initDefaultPool();
129                splitPool.execute(new Runnable() {
130                        @Override
131                        public void run() {
132                                SeekableByteChannel ch = null;
133                                int partIndex = -1;
134                                long fileSize = 0L;
135                                try {
136                                        Path originPath = Paths.get(origin.toURI());
137                                        Thread.currentThread().setName(String.format("DiskFile-split-%s", origin.getName()));
138                                        ch = Files.newByteChannel(originPath, StandardOpenOption.READ);
139                                        ByteBuffer bf = ByteBuffer.allocate(partSize);
140                                        while (ch.read(bf) != -1) {
141                                                bf.flip();
142                                                partIndex += 1;
143                                                List<Byte> parts = new ArrayList<Byte>();
144                                                while (bf.hasRemaining()) {
145                                                        parts.add(bf.get());
146                                                }
147                                                int partsSize = parts.size();
148                                                fileSize += partsSize;
149                                                byte[] part = new byte[partsSize];
150                                                for (int i = 0; i < partsSize; i++) {
151                                                        part[i] = parts.get(i);
152                                                }
153                                                filePart.process(partIndex, part);
154                                                bf.clear();
155                                        }
156                                        filePart.completed(partIndex, getModifyTimeForClock(), fileSize);
157                                } catch (Exception e) {
158                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
159                                } finally {
160                                        if (ch != null) {
161                                                try {
162                                                        ch.close();
163                                                } catch (IOException e) {
164                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
165                                                }
166                                        }
167                                        filePart.ended(partIndex, fileSize);
168                                }
169                        }
170                });
171        }
172
173        @Override
174        public void copyTo(String toPath) throws IOException {
175                Path originPath = Paths.get(origin.toURI());
176                Path destPath = Paths.get(toPath);
177                LoggerFactory.getLogger(DiskFile.class).debug("CopyTo: {} -> {}", originPath, destPath);
178                Files.copy(originPath, destPath);
179        }
180
181        @Override
182        public void moveTo(String toPath) throws IOException {
183                Path originPath = Paths.get(origin.toURI());
184                Path destPath = Paths.get(toPath);
185                LoggerFactory.getLogger(DiskFile.class).debug("MoveTo: {} -> {}", originPath, destPath);
186                if (Files.exists(destPath)) {
187                        Files.delete(destPath);
188                }
189                Files.move(originPath, destPath);
190        }
191
192        public RemoteFile writeToRemote(RemoteFile remoteFile) throws IOException {
193                return writeToRemote(remoteFile, new Runnable() {
194                        @Override
195                        public void run() {
196
197                        }
198                });
199        }
200
201        public RemoteFile writeToRemote(RemoteFile remoteFile, Runnable completedCallback) throws IOException {
202            if(remoteFile.exists(true)){
203                throw new IOException(String.format("The remote file '%s' already exists.", remoteFile.getPath()));
204                } else {
205                        remoteFile.copyFrom(copyStructureOnly, this);
206                        if (isDir()) {
207                                startFullAsync(0, this.getPath() + "/", remoteFile.getPath() + "/", completedCallback, null);
208                        }
209                }
210                return remoteFile;
211        }
212
213        protected boolean isLogicModify() {
214                boolean b = false;
215                try {
216                        String tmpPath = toModifyTmpFilePath();
217
218                        if (tmpPath == null)
219                                return false;
220
221                        DiskFile df = null;
222                        Path logicPath = Paths.get(tmpPath);
223                        if (Files.exists(logicPath)) {
224                                df = new DiskFile(tmpPath);
225                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
226                                Timestamp dfModifyTime = df.getModifyTimeForClock();
227
228                                if (dfModifyTime == null)
229                                        return false;
230
231                                long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
232                                
233                                b = diffMs < LOGIC_TIMEOUT_MS;
234                        }
235                } catch (Exception e) {
236                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
237                }
238                return b;
239        }
240
241        protected DiskFile logicModify() {
242                return logicModify(null);
243        }
244
245        protected DiskFile logicModify(String msg) {
246                try {
247                        if (getParentFile().exists()) {
248                                String tmpLogicPath = toModifyTmpFilePath();
249
250                                if (tmpLogicPath == null)
251                                        return null;
252
253                                Path logicPath = Paths.get(tmpLogicPath);
254                                DiskFile df = new DiskFile(tmpLogicPath);
255                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
256
257                                if (msg == null) {
258                                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
259                                                        false);
260                                } else {
261                                        df.write(msg, false);
262                                }
263
264                                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
265                                return df;
266                        }
267                } catch (Exception e) {
268                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
269                }
270                return null;
271        }
272
273        protected void removeLogicModify() {
274                try {
275                        String tmpPath = toModifyTmpFilePath();
276                        if (tmpPath != null) {
277                                DiskFile df = new DiskFile(tmpPath);
278                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
279                                if (df.exists()) {
280                                        df.delete();
281                                }
282                        }
283                } catch (Exception e) {
284                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
285                }
286        }
287
288        protected boolean isLogicAccess() {
289                boolean b = false;
290                try {
291                        String tmpPath = toAccessTmpFilePath();
292
293                        if (tmpPath == null)
294                                return false;
295
296                        DiskFile df = null;
297                        Path logicPath = Paths.get(tmpPath);
298                        if (Files.exists(logicPath)) {
299                                df = new DiskFile(tmpPath);
300                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
301                                Timestamp dfModifyTime = df.getModifyTimeForClock();
302                                
303                                long ltms = df.isFile() ? LOGIC_TIMEOUT_MS*2 : LOGIC_TIMEOUT_MS;
304
305                                if (dfModifyTime == null)
306                                        return false;
307
308                                long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
309                                
310                                b = diffMs < ltms;
311                        }
312                } catch (Exception e) {
313                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
314                }
315                return b;
316        }
317
318        protected DiskFile logicAccess() {
319                return logicAccess(null);
320        }
321
322        protected DiskFile logicAccess(String msg) {
323                try {
324                        if (getParentFile().exists()) {
325                                String tmpLogicPath = toAccessTmpFilePath();
326                                if (tmpLogicPath == null)
327                                        return null;
328
329                                Path logicPath = Paths.get(tmpLogicPath);
330                                DiskFile df = new DiskFile(tmpLogicPath);
331                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
332                                if (msg == null) {
333                                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
334                                                        false);
335                                } else {
336                                        df.write(msg, false);
337                                }
338
339                                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
340                                return df;
341                        }
342                } catch (Exception e) {
343                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
344                }
345                return null;
346        }
347
348        protected void removeLogicAccess() {
349                try {
350                        String tmpPath = toAccessTmpFilePath();
351                        if (tmpPath != null) {
352                                DiskFile df = new DiskFile(tmpPath);
353                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
354                                if (df.exists()) {
355                                        df.delete();
356                                }
357                        }
358                } catch (Exception e) {
359                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
360                }
361        }
362        
363        protected boolean isLogicCheck() {
364                boolean b = false;
365                try {
366                        String tmpPath = toCheckTmpFilePath();
367
368                        if (tmpPath == null)
369                                return false;
370
371                        DiskFile df = null;
372                        Path logicPath = Paths.get(tmpPath);
373                        if (Files.exists(logicPath)) {
374                                df = new DiskFile(tmpPath);
375                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
376                                Timestamp dfModifyTime = df.getModifyTimeForClock();
377
378                                if (dfModifyTime == null)
379                                        return false;
380
381                                long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
382                                
383                                b = diffMs < LOGIC_CHECK_TIMEOUT_MS;
384                        }
385                } catch (Exception e) {
386                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
387                }
388                return b;
389        }
390
391        protected DiskFile logicCheck() {
392                return logicCheck(null);
393        }
394
395        public DiskFile logicCheck(String msg) {
396                try {
397                        if (getParentFile().exists()) {
398                                String tmpLogicPath = toCheckTmpFilePath();
399                                if (tmpLogicPath == null)
400                                        return null;
401
402                                Path logicPath = Paths.get(tmpLogicPath);
403                                DiskFile df = new DiskFile(tmpLogicPath);
404                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
405                                if (msg == null) {
406                                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
407                                                        false);
408                                } else {
409                                        df.write(msg, false);
410                                }
411
412                                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
413                                return df;
414                        }
415                } catch (Exception e) {
416                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
417                }
418                return null;
419        }    
420
421        @Override
422        public boolean beforeDelete(boolean realDeleted) {
423                return true;
424        }
425        
426        @Override
427        public void afterDelete(boolean realDeleted) {
428            
429        }       
430        
431        @Override
432        public boolean beforeMkdirs() {
433            return true;
434        }
435        
436        @Override
437        public void afterMkdirs() {
438            
439        }
440        
441        @Override
442        public boolean beforeCreateLink(String target) {
443            return true;
444        }
445        
446        @Override
447        public void afterCreateLink(String target) {
448            
449        }
450        
451        @Override
452        public boolean beforeWrite(){
453            return true;
454        }
455        
456        @Override
457        public void afterWrite(){
458            
459        }
460
461        @Override
462        public boolean delete() throws IOException {
463                return deleteFrom();
464        }
465
466        private boolean deleteFrom() throws IOException {
467                boolean allowed = beforeDelete(true);
468                if (allowed) {
469                        LoggerFactory.getLogger(DiskFile.class).debug("Delete: {}", getPath());
470                        if (isFile()) {
471                                Path originPath = Paths.get(origin.toURI());
472                                boolean exists = Files.exists(originPath);
473                                if (exists) {
474                                        Files.deleteIfExists(Paths.get(origin.toURI()));
475                                        afterDelete(true);
476                                        return true;
477                                }
478                        }
479                        if (isDir()) {
480                                Path originPath = Paths.get(origin.toURI());
481                                boolean exists = Files.exists(originPath);
482                                if (exists) {
483                                        Files.walk(originPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
484                                        afterDelete(true);
485                                        return true;
486                                }
487                        }
488                        if (isLink()) {
489                                Path originPath = Paths.get(origin.toURI());
490                                boolean exists = Files.exists(originPath);
491                                if (exists) {
492                                        Files.deleteIfExists(Paths.get(origin.toURI()));
493                                        afterDelete(true);
494                                        return true;
495                                }
496                        }
497                }
498                return false;
499        }
500
501        @Override
502        public boolean isFile() throws IOException {
503                return this.origin.isFile() && !isLink();
504        }
505
506        @Override
507        public boolean isDirectory() throws IOException {
508                return this.origin.isDirectory() && !isLink();
509        }
510
511        @Override
512        public boolean isLink() throws IOException {
513                return Files.isSymbolicLink(Paths.get(this.origin.toURI()));
514        }
515
516        @Override
517        public boolean exists() throws IOException {
518                return this.origin.exists();
519        }
520
521        @Override
522        public boolean mkdirs() throws IOException {
523            boolean allowed = beforeMkdirs();
524            if(allowed){
525                LoggerFactory.getLogger(DiskFile.class).debug("Mkidrs: {}", getPath());
526                boolean b = this.origin.mkdirs();
527                if (b && modifyTimeMs > 0) {
528                        setModifyTime(new Timestamp(modifyTimeMs));
529                }
530                afterMkdirs();
531                return b;
532            }
533            return false;
534        }
535
536    @Override
537        public boolean write(byte[] data, boolean append) throws IOException {
538            
539            boolean allowed = beforeWrite();
540            
541            if(!allowed) return false;
542
543                if (data == null)
544                        data = new byte[] {};
545
546                try {
547
548                        OpenOption[] options = null;
549                        if (append) {
550                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
551                                                StandardOpenOption.APPEND };
552                        } else {
553                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
554                                                StandardOpenOption.TRUNCATE_EXISTING };
555                        }
556
557                        File parent = origin.getParentFile();
558                        if (!parent.exists()) {
559                                parent.mkdirs();
560                        }
561
562                        Path path = Paths.get(origin.toURI());
563                        channel = FileChannel.open(path, options);
564                        lock = channel.lock();
565                        if (lock != null) {
566                                ByteBuffer buf = ByteBuffer.wrap(data);
567                                channel.write(buf);
568                                if (modifyTimeMs > 0) {
569                                        setModifyTime(new Timestamp(modifyTimeMs));
570                                }
571                        }
572                        return true;
573                } catch (IOException e) {
574                        throw e;
575                } finally {
576                        close();
577                }
578        }
579
580        public boolean manualOpen(boolean append) throws IOException, FileAlreadyExistsException {
581
582            boolean allowed = beforeWrite();
583            
584            if(!allowed) return false;
585            
586                OpenOption[] options = null;
587                if (append) {
588                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
589                                        StandardOpenOption.APPEND };
590                } else {
591                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
592                                        StandardOpenOption.TRUNCATE_EXISTING };
593                }
594                Path path = Paths.get(origin.toURI());
595                File parent = origin.getParentFile();
596                if (parent != null && !parent.exists() && !parent.mkdirs()) {
597                        throw new IOException(String.format("Failed to create parent directory: %s", parent));
598                }
599                if (!Files.exists(path)) {
600                        try {
601                                Files.createFile(path);
602                        } catch (FileAlreadyExistsException e) {
603                                throw e;
604                        }
605                }
606                channel = FileChannel.open(path, options);
607                lock = channel.lock();
608                
609                return true;
610        }
611
612        public void manualWrite(byte[] data) throws IOException {
613                if (data == null)
614                        data = new byte[] {};
615
616                if (lock != null) {
617                        ByteBuffer buf = ByteBuffer.wrap(data);
618                        channel.write(buf);
619                        if (modifyTimeMs > 0) {
620                                setModifyTime(new Timestamp(modifyTimeMs));
621                        }
622                }
623        }
624
625        public void manualClose() throws IOException {
626                close();
627        }
628
629        @Override
630        public boolean write(byte[] data) throws IOException {
631                return write(data, false);
632        }
633
634        @Override
635        public boolean write(String data, boolean append) throws IOException {
636                return write(data.getBytes(CHARSET), append);
637        }
638
639        public boolean write(String data, String charset,boolean append) throws IOException {
640                return write(data.getBytes(charset), append);
641        }
642        
643        @Override
644        public boolean write(String data) throws IOException {
645                return write(data.getBytes(CHARSET));
646        }
647        
648        public boolean write(String data,String charset) throws IOException {
649                return write(data.getBytes(charset));
650        }       
651
652        @Override
653        public boolean createLink(String target) throws IOException {
654            boolean allowed = beforeCreateLink(target);
655            if(allowed){
656                LoggerFactory.getLogger(DiskFile.class).debug("CreateLink: {}", getPath());
657                try {
658                        Path link = Paths.get(origin.toURI());
659                        Path targetPath = Paths.get(target);
660                        Files.createSymbolicLink(link, targetPath);
661                        if (modifyTimeMs > 0) {
662                                setModifyTime(new Timestamp(modifyTimeMs));
663                        }
664                        afterCreateLink(target);
665                        return true;
666                } catch (IOException e) {
667                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
668                }
669            }
670                return false;
671        }
672
673        @Override
674        public String readLink() throws IOException {
675                Path link = Paths.get(origin.toURI());
676                return Files.readSymbolicLink(link).toString();
677        }
678
679        @Override
680        public byte[] readAllBytes() throws IOException {
681                return readAllBytesFrom();
682        }
683        
684        private byte[] readAllBytesFrom() throws IOException {
685                if (isLink()) {
686                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
687                }
688                if (isDir()) {
689                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
690                }
691                if (!exists()) {
692                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
693                }
694                return Files.readAllBytes(Paths.get(getPath()));
695        }
696
697        @Override
698        public String readAllString() throws IOException {
699                return readAllString(BaseFile.CHARSET);
700        }
701        
702        public String readAllString(String charset) throws IOException {
703                ByteArrayOutputStream baos = null;
704                try {
705                        byte[] bytes = readAllBytes();
706                        baos = new ByteArrayOutputStream();
707                        baos.write(bytes);
708                        baos.flush();
709                        return baos.toString(charset);
710                } finally {
711                        if (baos != null) {
712                                try {
713                                        baos.close();
714                                } catch (IOException e) {
715                                        Logger.systemError(DiskFile.class, e.getMessage(), e);
716                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
717                                }
718                        }
719                }
720        }       
721
722        @Override
723        public long size() throws IOException {
724                if (isFile()) {
725                        Path path = Paths.get(origin.getAbsolutePath());
726                        return Files.size(path);
727                }
728                return 0L;
729        }
730
731        private void close() {
732                if (lock != null) {
733                        try {
734                                lock.release();
735                        } catch (IOException e) {
736                                Logger.systemError(DiskFile.class, e.getMessage(), e);
737                        }
738                        try {
739                                lock.close();
740                        } catch (IOException e) {
741                                Logger.systemError(DiskFile.class, e.getMessage(), e);
742                        } finally {
743                                lock = null;
744                        }
745                }
746
747                if (channel != null) {
748                        try {
749                                channel.close();
750                        } catch (IOException e) {
751                                Logger.systemError(DiskFile.class, e.getMessage(), e);
752                        } finally {
753                                channel = null;
754                        }
755                }
756                afterWrite();
757        }
758
759        @Override
760        public boolean complete() throws IOException {
761                close();
762                return true;
763        }
764
765        public void setModifyTimeMs(long modifyTimeMs) throws IOException {
766                this.modifyTimeMs = modifyTimeMs;
767        }
768
769        public void setModifyTimeMsFromClock(long modifyTimeMsFromClock) throws IOException {
770                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
771                String localTzId = Calendar.getInstance().getTimeZone().getID();
772                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, new Timestamp(modifyTimeMsFromClock));
773                this.modifyTimeMs = fileLocalTimestamp.getTime();
774        }
775
776        @Override
777        public void setModifyTime(Timestamp modifyTime) throws IOException {
778                if (exists()) {
779                        Path originPath = Paths.get(origin.toURI());
780                        Files.setLastModifiedTime(originPath, FileTime.fromMillis(modifyTime.getTime()));
781                }
782        }
783
784        public void setModifyTimeFromClock(Timestamp modifyTimeFromClock) throws IOException {
785                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
786                String localTzId = Calendar.getInstance().getTimeZone().getID();
787                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, modifyTimeFromClock);
788                setModifyTime(fileLocalTimestamp);
789        }
790
791        @Override
792        public Timestamp getModifyTime() throws IOException {
793                Path originPath = Paths.get(origin.toURI());
794                if (Files.exists(originPath)) {
795                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
796                        FileTime fileTime = attr.lastModifiedTime();
797                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
798                        return fileLocalTimestamp;
799                }
800                return null;
801        }
802
803        public Timestamp getModifyTimeForClock() throws IOException {
804                Path originPath = Paths.get(origin.toURI());
805                if (Files.exists(originPath)) {
806                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
807                        FileTime fileTime = attr.lastModifiedTime();
808                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
809                        String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
810                        String localTzId = Calendar.getInstance().getTimeZone().getID();
811                        Timestamp fileRemoteTimestamp = timeZoneConver(remoteTzId, localTzId, fileLocalTimestamp);
812                        return fileRemoteTimestamp;
813                }
814                return null;
815        }
816
817        public void startFullAsync(long timerMs, String replaceTo) throws IOException {
818                startFullAsync(timerMs, getPath() + "/", replaceTo, null, null);
819        }
820
821        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback) throws IOException {
822                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, null);
823        }
824
825        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback,
826                        Runnable checkDeleteCompletedCallback) throws IOException {
827                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, checkDeleteCompletedCallback);
828        }
829
830        private void startFullAsync(long timerMs,String replaceRootPath, String replaceTo,
831                        Runnable completedCallback, Runnable checkDeleteCompletedCallback) throws IOException {
832                            
833                if(!RemoteFile.FIRST_LOADED){
834                    RemoteFile firstLoaded = new RemoteFile(replaceTo);
835                }           
836                
837                Runnable runnable = new Runnable() {
838                        @Override
839                        public void run() {
840                                try {
841                                        startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
842                                } catch (Exception e) {
843                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
844                                }
845                        }
846                };
847                Executors.newFixedThreadPool(1).execute(runnable);
848
849                runnable = new Runnable() {
850                        @Override
851                        public void run() {
852                                try {
853                                        startFullSyncForDelete(-1,timerMs,replaceTo, replaceRootPath,
854                                                        checkDeleteCompletedCallback);
855                                } catch (Exception e) {
856                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
857                                }
858                        }
859                };
860                Executors.newFixedThreadPool(1).execute(runnable);
861        }
862        
863        private synchronized void startFullSyncForDelete(final Integer filePartDataTable,final long timerMs,final String replaceTo,
864                        final String replaceRootPath, final Runnable completedCallback) throws IOException {
865                DiskFile the = this;
866                RemoteFile rootDir = null;
867                if (!CommonTools.isBlank(replaceTo)) {
868                                rootDir = new RemoteFile(replaceTo);
869                }
870                if (rootDir == null) {
871                        throw new IOException("The root folder is null.");
872                }
873                RemoteFile _rootDir = rootDir;
874        Integer maxDataTable = rootDir.getMaxFilePartDataTable();
875        
876        Integer _filePartDataTable = filePartDataTable;
877        
878        if(filePartDataTable > maxDataTable) _filePartDataTable = -1;
879        
880        final Integer finalFilePartDataTable = _filePartDataTable;
881        
882            final CacheArray rows = new CacheArray();
883            
884                rows.filter(getScanDeleteFilter(rootDir,finalFilePartDataTable,timerMs,replaceTo,replaceRootPath,completedCallback));
885
886        Runnable runnable = new Runnable(){
887            @Override
888            public void run(){
889                try{
890                        _rootDir.listAllForScanDelete(finalFilePartDataTable,rows);
891                }catch(Exception e){
892                    LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
893                }
894            }
895        };
896        Executors.newFixedThreadPool(1).execute(runnable);
897        }       
898        
899        private CacheArrayFilter getScanDeleteFilter(final RemoteFile rootDir,final Integer filePartDataTable,final long timerMs,final String replaceTo,
900                        final String replaceRootPath, final Runnable completedCallback){
901                DiskFile the = this;
902                CacheArrayFilter filter = new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
903                        @Override
904                        public void execute(Integer index, Object o) {
905                
906                                try {
907                                        Map<String, Object> item = (Map<String, Object>) o;
908                                        Object id = item.get("id");
909                                        String parent = (String) item.get("file_parent_path");
910                                        String fileName = (String) item.get("file_name");
911                                        String pf = String.format("%s/%s", parent, fileName);
912                                        RemoteFile rf = new RemoteFile(pf);
913                                        Thread.currentThread().setName(String.format("DiskFile-startFullSyncForDelete-(DataTable=%s Index=%s)-%s",filePartDataTable,index,fileName));
914
915                                        if (memoryUsage() < RemoteFile.MAX_MEMORY_USAGE) {
916                                        String toDf = pf;
917                                        if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
918                                                toDf = pf.replaceFirst(replaceTo + "[/]*", replaceRootPath + "/");
919                                        }
920                                        
921                                        DiskFile df = new DiskFile(toDf);
922                                        df.copyAttrs(df, df.copyStructureOnly, df.syncRoot);
923                                        boolean isDebug = isDebug(rf);
924                                        boolean isRoot = rootDir.getPath().equals(rf.getPath());
925                                        boolean isSyncedHost = rf.isSyncedOnHostname(id);
926                                        boolean rfExists = rf.exists();
927    
928                                        boolean allowedForceDelete = rootDir.exists() && !isRoot && isSyncedHost && rfExists
929                                                        && !df.exists() && !df.isLogicModify();
930                                                        
931                                        if (allowedForceDelete) {
932                                                df.logicModify();
933                                                if (isDebug) {
934                                                        LoggerFactory.getLogger(DiskFile.class).mark("ForceRemoteDelete - {}", rf.getPath());
935                                                }
936                                                rf.forceDeleteAll(true);
937                                                df.removeLogicModify();
938                                        }
939                                        }else{
940                                            LoggerFactory.getLogger(DiskFile.class).warn("Over max memory usage 'MAX_MEMORY_USAGE={}'.",RemoteFile.MAX_MEMORY_USAGE);
941                                        }
942                                } catch (Exception e) {
943                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
944                                }
945                        }
946
947                        @Override
948                        public void completed(Integer size) {
949                                if (isStopSync()) {
950                                        LoggerFactory.getLogger(DiskFile.class).mark("Stoped start full sync for delete.");
951                                }
952                                if (timerMs <= 0) {
953                                        LoggerFactory.getLogger(DiskFile.class).mark("Completed start full sync for delete.");
954                                }
955
956                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
957                                
958                                if (isRootCompleted && completedCallback != null) {
959                                        completedCallback.run();
960                                }
961                                
962                                if (!isStopSync() && timerMs > 0) {
963                                    while(true){
964                                        
965                                        if(isStopSync()) break;
966                                        
967                                        try {
968                                                Thread.sleep(timerMs);
969                                        } catch (InterruptedException e) {
970                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
971                                        }
972                                        boolean allowed = checkUsage(rootDir);
973                                        if(allowed){
974                                                try {
975                                                    if(!isStopSync()){
976                                                        startFullSyncForDelete(filePartDataTable + 1,timerMs, replaceTo, replaceRootPath,
977                                                                        completedCallback);
978                                                    }
979                                                } catch (IOException e) {
980                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
981                                                }
982                                                break;
983                                        }
984                                    }
985                                }
986                        }
987                };
988                return filter;
989        }
990
991        private void startFullAsyncThread(final long timerMs, String replaceRootPath,
992                        String replaceTo, final Runnable completedCallback) {
993                stopSync = false;
994                this.setCopyStructureOnly(RemoteFile.COPY_STRUCTURE_ONLY);
995                final DiskFile the = this;
996                Thread.currentThread()
997                                                .setName(String.format("DiskFile-startFullAsyncThread-%s",the.getOrigin().getName()));
998                replaceRootPath = replacePath(replaceRootPath);
999                replaceTo = replacePath(replaceTo);
1000                DiskFile rootDisk = new DiskFile(replaceRootPath);
1001                boolean rootExists = false;
1002                try{
1003                    rootExists = rootDisk.exists();
1004                }catch(Exception e){
1005                    LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(),e);
1006                }
1007                if(!rootExists){
1008                    Logger.systemError(DiskFile.class,"The root path '{}' does not exist.",rootDisk.getPath());
1009                }
1010        boolean isRoot = the.getPath().equals(rootDisk.getPath());
1011        LoggerFactory.getLogger(DiskFile.class).debug("IsRoot: {} - {}",isRoot,the.getPath());
1012        LoggerFactory.getLogger(DiskFile.class).debug("CacheArrayUsage: {}",CacheArray.getUsage());
1013                if (rootExists && (isRoot || !the.isLogicAccess())) {
1014                        if(!isRoot) the.logicAccess();
1015                        
1016                        DirectoryStream<Path> stream = null;
1017                        try {
1018                                if (syncRoot == null) {
1019                                        File replaceRootFile = new File(replaceRootPath);
1020                                        if (replaceRootFile.getParent() == null) {
1021                                                throw new IOException("Cannot sync the root path.");
1022                                        }
1023                                        String replaceRootParent = replacePath(replaceRootFile.getParent());
1024                                        syncRoot = new File(String.format("%s/%s", replaceRootParent, replaceRootFile.getName()));
1025                                }
1026
1027                                if (!CommonTools.isBlank(replaceTo)) {
1028                                        RemoteFile rootDir = new RemoteFile(replaceTo);
1029                                        if (!rootDir.exists()) {
1030                                                rootDir.setModifyTime(this.getModifyTimeForClock());
1031                                                rootDir.mkdirs();
1032                                        }
1033                                }
1034                                Path originPath = Paths.get(origin.toURI());
1035                                CacheArray rows = new CacheArray();
1036                                CacheArrayFilter filter = getFullAsyncFilter(the, timerMs, replaceRootPath, replaceTo,
1037                                                completedCallback);
1038                                rows.filter(filter);
1039                                stream = Files.newDirectoryStream(originPath);
1040                                for (Path p : stream) {
1041                                        if (p != null) {
1042                                            File f = p.toFile();
1043                                            boolean isFile = f.isFile();
1044                                            if(isFile){
1045                                                boolean isWritingDf = f.getName().matches(TMP_WRITING_SWP);
1046                                                if(isWritingDf){
1047                                BasicFileAttributes attributes = Files.readAttributes(p, BasicFileAttributes.class);
1048                                FileTime creationTime = attributes.creationTime();
1049                                boolean timeouted = (Calendar.getInstance().getTimeInMillis() - creationTime.toMillis()) > (RemoteFile.BATCH_SIZE*LOGIC_TIMEOUT_MS);
1050                                if(timeouted){
1051                                    try{
1052                                        boolean exists = Files.exists(p);
1053                                                        if (exists) {
1054                                                                Files.deleteIfExists(p);
1055                                                        }
1056                                    }catch(Exception e){
1057                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1058                                    }
1059                                }
1060                                                }
1061                                            }
1062                                                boolean allowedPath = the.checkSyncPathAvailable(p.toString());
1063                                                if(!allowedPath){
1064                                                    LoggerFactory.getLogger(DiskFile.class).debug("Ignored: {}",p.toString());
1065                                                }
1066                                                if (allowedPath) {
1067                                                        rows.add(p);
1068                                                }
1069                                        }
1070                                }
1071                                rows.add(null);
1072                        } catch (Exception e) {
1073                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1074                                removeLogicAccess();
1075                                if (!(e instanceof NoSuchFileException)) {
1076                                        if (!isStopSync() && timerMs > 0) {
1077                                                try {
1078                                                        Thread.sleep(timerMs);
1079                                                } catch (InterruptedException ee) {
1080                                                        LoggerFactory.getLogger(DiskFile.class).error(ee.getMessage(), ee);
1081                                                }
1082                                                if(!isStopSync()){
1083                                                    startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
1084                                                }
1085                                        }
1086                                }
1087                        } finally {
1088                                if (stream != null) {
1089                                        try {
1090                                                stream.close();
1091                                        } catch (IOException e) {
1092                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1093                                        }
1094                                }
1095                        }
1096                }
1097        }
1098
1099        private CacheArrayFilter getFullAsyncFilter(DiskFile the, long timerMs,
1100                        String replaceRootPath, String replaceTo, Runnable completedCallback) {
1101
1102                return new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
1103                        RemoteFile rf = null;
1104            final CacheArrayFilter theFilter = this;    
1105                        @Override
1106                        public void completed(Integer size) {
1107                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1108                                
1109                                if(!isRootCompleted) the.removeLogicAccess();
1110                                
1111                                if (isRootCompleted) {
1112                                        try {
1113                        if (rf != null && the.isStopSync() && isDebug(rf)) {
1114                                                LoggerFactory.getLogger(DiskFile.class).mark("StopedRootScan - {}", the.getPath());
1115                                        }                                                               
1116                                                if (!the.isStopSync() && timerMs > 0) {
1117                                                        Thread.sleep(timerMs);
1118                                                if (!the.isStopSync()) {
1119                                                if (rf != null && isDebug(rf)) {
1120                                                        LoggerFactory.getLogger(DiskFile.class).mark("RootScaning - {}", the.getPath());
1121                                                }                                                   
1122                                                            the.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1123                                                                            completedCallback);
1124                                                }
1125                                                }
1126                                                if (completedCallback != null) {
1127                                                        completedCallback.run();
1128                                                }
1129                                        } catch (Exception e) {
1130                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1131                                        }
1132                                }
1133                        }
1134
1135                        @Override
1136                        public void execute(Integer index, Object o) {
1137                DiskFile df = null;
1138                try{
1139                                Path p = (Path) o;
1140                                String pf = replacePath(p.toFile().getAbsolutePath());
1141                                Thread.currentThread()
1142                                                .setName(String.format("DiskFile-getFullAsyncFilter-%s", p.toFile().getParentFile().getName()));
1143                                String toRf = pf;
1144                                if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
1145                                        toRf = pf.replaceFirst(replaceRootPath + "[/]*", replaceTo + "/");
1146                                }
1147    
1148                                df = new DiskFile(pf);
1149                                df.copyAttrs(df, the.copyStructureOnly, the.syncRoot);
1150                                rf = new RemoteFile(toRf);
1151    
1152                    boolean isRoot = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1153                    
1154                    boolean allowedConn = checkUsage(rf);
1155                                        String fileName = p.toFile().getName();
1156                                        boolean isDebug = isDebug(rf);
1157
1158                                        boolean timeouted = rf.isTimeout();
1159                                        if (timeouted && RemoteFile.addQueuePathMapping(rf.getPath()) && allowedConn) {
1160                                                if (isDebug) {
1161                                                        LoggerFactory.getLogger(DiskFile.class).mark("ForceDelete(Timeouted) - {}",
1162                                                                        rf.getPath());
1163                                                }                                                           
1164                                                rf.forceDeleteFile(false);
1165                                                RemoteFile.removeQueuePathMapping(rf.getPath());
1166                                        }
1167                                        Timestamp modifyTime = df.getModifyTimeForClock();
1168                                        boolean allowedModifyTime = modifyTime != null;
1169                                        boolean allowed = DiskFile.checkQueuePathMapping() && !isStopSync() && allowedModifyTime && allowedConn;
1170                                        
1171                                        if(isDebug){
1172                                            debugLog("CheckUsage",rf,this);
1173                                        }
1174                                        
1175                                        if(!isRoot && allowed){
1176                                            allowed = !df.isLogicCheck();
1177                                        }
1178                                        
1179                                        if (allowed) {
1180                                            
1181                                            if(!isRoot) df.logicCheck();
1182                                            
1183                                                rf.setModifyTime(modifyTime);
1184                                                long dfModifyTimeMs = modifyTime.getTime();
1185                                                boolean blocked = false;
1186                                                boolean isRealDelete = rf.isLastOperateRealDelete();
1187
1188                                                if (isRealDelete) {
1189                                                        String clientHostname = CommonTools.getHostname();
1190                                                        String delHostname = rf.getLastSourceHostname();
1191                                                        boolean isOwnerDel = clientHostname.equals(delHostname);
1192                                                        if (isOwnerDel) {
1193                                                                Timestamp deletedTime = rf.getLastOperateTime();
1194                                                                if (deletedTime != null) {
1195                                                                        boolean waiting = new Clock().getTime() - deletedTime.getTime() <= LOGIC_TIMEOUT_MS;
1196                                                                        if (waiting) {
1197                                                                                blocked = true;
1198                                                                        }
1199                                                                }
1200                                                        } else {
1201                                                                Timestamp deletedTime = rf.getLastOperateTime();
1202                                                                if (deletedTime != null) {
1203                                                                        boolean olded = modifyTime.getTime() < deletedTime.getTime();
1204                                                                        if (olded) {
1205                                                                                blocked = true;
1206                                                                        }
1207                                                                }
1208                                                        }
1209                                                }
1210
1211                                                boolean haveParent = rf.getParentFile().exists();
1212                                                if (!blocked && haveParent) {    
1213                                                        if (df.isLink()) {
1214                                                                df.logicAccess();
1215                                                                if (rf.exists()) {
1216                                                                        if (rf.isLink()) {
1217                                                                                long rfModifyTimeMs = rf.getModifyTime().getTime();
1218                                                                                boolean changedTime = dfModifyTimeMs > rfModifyTimeMs;
1219                                                                                boolean changedValue = !rf.readLink().equals(df.readLink());
1220                                                                                boolean changed = changedTime && changedValue;
1221                                                                                if (changed) {
1222                                                                                RemoteFile prf = rf.getParentFile();
1223                                                                                if(prf != null && prf.exists()){                                                                                    
1224                                                                                        if (isDebug) {
1225                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1226                                                                                                                rf.getPath());
1227                                                                                        }
1228                                                                                        rf.forceDeleteLink(false);
1229                                                                                        rf.createLink(df.readLink());
1230                                                                                }
1231                                                                                }
1232                                                                        } else {
1233                                                                        RemoteFile prf = rf.getParentFile();
1234                                                                        if(prf != null && prf.exists()){                                                                            
1235                                                                                if (isDebug) {
1236                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1237                                                                                                        rf.getPath());
1238                                                                                }
1239                                                                                rf.forceDeleteLink(false);
1240                                                                                rf.createLink(df.readLink());
1241                                                                        }
1242                                                                        }
1243                                                                } else {
1244                                                                        RemoteFile prf = rf.getParentFile();
1245                                                                        if(prf != null && prf.exists()){
1246                                                                        if (isDebug) {
1247                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}", rf.getPath());
1248                                                                        }       
1249                                                                            rf.createLink(df.readLink());
1250                                                                        }
1251                                                                }
1252                                                                df.removeLogicAccess();
1253                                                        } else if (df.isFile()) {
1254                                                                if (!df.isLogicModify() && !df.isLogicAccess()) {
1255                                                                        if (rf.exists()) {
1256                                                                                if (rf.isFile()) {
1257                                                                                        long rfModifyTimeMs = rf.getModifyTime().getTime();
1258                                                                                        if (dfModifyTimeMs > rfModifyTimeMs) {
1259                                                                                            RemoteFile prf = rf.getParentFile();
1260                                                                                            if(prf != null && prf.exists()){
1261                                                                                                if (DiskFile.addQueuePathMapping(df.getPath())) {
1262                                                                                                        if (isDebug) {
1263                                                                                                                LoggerFactory.getLogger(DiskFile.class)
1264                                                                                                                                .mark("RemoteQueuing - {}", df.getPath());
1265                                                                                                        }
1266                                                                                                        df.logicAccess();
1267                                                                                                        rf.forceDeleteFile(false);
1268                                                                                                        df.writeToRemote(rf);
1269                                                                                                }
1270                                                                                            }
1271                                                                                        }
1272                                                                                } else {
1273                                                                                        rf.forceDeleteFile(false);
1274                                                                                }
1275                                                                        } else {
1276                                                                            RemoteFile prf = rf.getParentFile();
1277                                                                            if(prf != null && prf.exists()){
1278                                                                                if (DiskFile.addQueuePathMapping(df.getPath())) {
1279                                                                                        if (isDebug) {
1280                                                                                                LoggerFactory.getLogger(DiskFile.class)
1281                                                                                                                .mark("RemoteQueuing - {}", df.getPath());
1282                                                                                        }
1283                                                                                        df.logicAccess();
1284                                                                                        df.writeToRemote(rf);
1285                                                                                }
1286                                                                            }
1287                                                                        }
1288                                                                }
1289                                                        } else if (df.isDir()) {
1290                                                                boolean isCompleted = !df.isLogicAccess();
1291                                                                if (isStopSync() && isDebug) {
1292                                                                        LoggerFactory.getLogger(DiskFile.class).mark("Stopping queue");
1293                                                                }
1294                                                                if (!isStopSync() && isCompleted) {
1295                                                                        if (isDebug) {
1296                                                                                LoggerFactory.getLogger(DiskFile.class).debug("Scaning - {}", df.getPath());
1297                                                                        }
1298                                                                        boolean allowedPoolSize = theFilter.getCacheArray().getUsingPoolSize() < theFilter.getCacheArray().getMaxPoolSize();
1299                                                                        if (rf.exists()) {
1300                                                                                if (rf.isDir()) {
1301                                                                                    allowedConn = allowedPoolSize && checkUsage(rf);
1302                                                                                    if(isRoot || allowedConn){
1303                                                                                        df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1304                                                                                                        completedCallback);
1305                                                                                    }
1306                                                                                } else {
1307                                                                                        if (rf.isDir())
1308                                                                                                rf.forceDeleteDir(false);
1309
1310                                                                                        if (rf.isFile())
1311                                                                                                rf.forceDeleteFile(false);
1312
1313                                                                                        if (rf.isLink())
1314                                                                                                rf.forceDeleteLink(false);
1315                                            
1316                                            RemoteFile prf = rf.getParentFile();
1317                                            if(isRoot || (prf != null && prf.exists())){
1318                                                                                        if (isDebug) {
1319                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("ReCreateRemoteDir - {}",
1320                                                                                                                rf.getPath());
1321                                                                                        }                                                
1322                                                                                        rf.mkdirs();
1323                                                                                        allowedConn = allowedPoolSize && checkUsage(rf);
1324                                                if(isRoot || allowedConn){
1325                                                                                                df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1326                                                                                                                completedCallback);
1327                                                }
1328                                            }
1329                                                                                }
1330                                                                        } else {
1331                                                                                RemoteFile prf = rf.getParentFile();
1332                                                                                if(isRoot || (prf != null && prf.exists())){
1333                                                                                if (isDebug) {
1334                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("CreateRemoteDir - {}",
1335                                                                                                        rf.getPath());
1336                                                                                }                                                                                   
1337                                                                                rf.mkdirs();
1338                                                                            allowedConn = allowedPoolSize && checkUsage(rf);
1339                                            if(isRoot || allowedConn){
1340                                                                                    df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1341                                                                                                    completedCallback);
1342                                            }
1343                                                                                }
1344                                                                        }
1345                                                                        if(!isRoot && !allowedPoolSize) df.removeLogicAccess();
1346                                                                }
1347                                                        }
1348                                                }
1349                                        }
1350                                } catch (Exception e) {
1351                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1352                                        if (df != null) {
1353                                                df.removeLogicAccess();
1354                                        }
1355                                }
1356                        }
1357
1358                };
1359        }
1360
1361        public void setCopyStructureOnly(boolean copyStructureOnly) {
1362                this.copyStructureOnly = copyStructureOnly;
1363        }
1364
1365        public boolean isCopyStructureOnly() {
1366                return this.copyStructureOnly;
1367        }
1368
1369        private boolean isViEditSwp(String fileName) {
1370                return fileName.matches(TMP_MATCHES_EDIT) || fileName.matches(TMP_MATCHES_VI_SWP);
1371        }
1372
1373        protected DiskFile copyAttrs(DiskFile source, boolean _copyStructureOnly, File _syncRoot) {
1374                source.setCopyStructureOnly(_copyStructureOnly);
1375                source.syncRoot = _syncRoot;
1376                return source;
1377        }
1378
1379        public File getParentFile() {
1380                return new File(getParent());
1381        }
1382
1383        protected boolean checkSyncPathAvailable(String path) {
1384            try{
1385                    return checkSyncPathAvailable(path, RemoteFile.SHOW_HIDDEN, RemoteFile.SHOW_LINK, RemoteFile.MAX_FILE_SZIE,
1386                        SYNC_PATH_ALLOWED,
1387                                SYNC_PATH_IGNORED);
1388            }catch(Exception e){
1389                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1390                return false;
1391            }
1392        }
1393
1394        protected static synchronized boolean addQueuePathMapping(String path) {
1395                Long addTimeMs = null;
1396        boolean allowed = checkQueuePathMapping();
1397        
1398        if(!allowed) return false;
1399
1400                addTimeMs = QUEUE_PATH_MAPPING.get(path);
1401
1402                if (addTimeMs == null) {
1403                        QUEUE_PATH_MAPPING.put(path, System.currentTimeMillis());
1404                        return true;
1405                }
1406
1407                return false;
1408        }
1409
1410        public static synchronized boolean checkQueuePathMapping() {
1411                Long addTimeMs = null;
1412                Map<String, Long> copyQueueMap = getQueuePathMapping();
1413                List<String> keyList = new ArrayList<String>(copyQueueMap.keySet());
1414                for (String key : keyList) {
1415                        addTimeMs = copyQueueMap.get(key);
1416                        if (addTimeMs != null) {
1417                                boolean timeout = (System.currentTimeMillis() - addTimeMs) >= LOGIC_TIMEOUT_MS;
1418
1419                                if (timeout) {
1420                                        QUEUE_PATH_MAPPING.remove(key);
1421                                }
1422                        }
1423                }
1424
1425                if (QUEUE_PATH_MAPPING.keySet().size() >= RemoteFile.MAX_QUEUE_SIZE)
1426                        return false;
1427
1428                return true;
1429        }
1430
1431        protected static synchronized void removeQueuePathMapping(String path) {
1432                QUEUE_PATH_MAPPING.remove(path);
1433        }
1434
1435        public static synchronized Map<String, Long> getQueuePathMapping() {
1436                return Collections.synchronizedMap(QUEUE_PATH_MAPPING);
1437        }
1438
1439        private static void debugLog(String point,RemoteFile rf,CacheArrayFilter filter){
1440                debugLog(DiskFile.class,point,rf,filter);
1441        }
1442        
1443}