001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import java.nio.file.Paths;
028import java.nio.file.Files;
029import java.io.IOException;
030import java.nio.file.OpenOption;
031import java.nio.file.StandardOpenOption;
032import java.nio.channels.FileChannel;
033import java.nio.channels.FileLock;
034import java.nio.file.Path;
035import java.nio.ByteBuffer;
036import com.killcoding.log.Logger;
037import java.net.URI;
038import java.util.Comparator;
039import java.util.List;
040import java.util.ArrayList;
041import java.nio.channels.SeekableByteChannel;
042import java.io.ByteArrayOutputStream;
043import java.nio.file.attribute.BasicFileAttributes;
044import java.nio.file.attribute.FileTime;
045import java.time.Instant;
046import java.util.Arrays;
047import java.io.ByteArrayInputStream;
048import java.sql.Timestamp;
049import java.util.stream.Stream;
050import java.util.stream.Collectors;
051import java.util.HashSet;
052import java.util.Set;
053import java.nio.file.DirectoryStream;
054import java.nio.file.CopyOption;
055import java.nio.file.LinkOption;
056import com.killcoding.tool.CommonTools;
057import com.killcoding.file.RemoteFile;
058import com.killcoding.log.LoggerFactory;
059import java.util.concurrent.Future;
060import com.killcoding.tool.ConfigProperties;
061import com.killcoding.datasource.Clock;
062import java.util.concurrent.Callable;
063import com.killcoding.file.DiskFile;
064import java.nio.file.attribute.FileAttribute;
065import java.text.SimpleDateFormat;
066import com.killcoding.cache.CacheArray;
067import com.killcoding.cache.CacheArrayFilter;
068import java.util.Map;
069import java.util.concurrent.ExecutorService;
070import java.util.concurrent.Executors;
071import java.nio.file.NoSuchFileException;
072import java.util.concurrent.ConcurrentHashMap;
073import java.util.Iterator;
074import java.util.Date;
075import java.time.ZonedDateTime;
076import java.time.LocalDateTime;
077import java.time.ZoneId;
078import java.time.format.DateTimeFormatter;
079import java.util.Calendar;
080import java.util.TimeZone;
081import java.text.ParseException;
082import java.util.Collections;
083import java.util.HashMap;
084import java.nio.file.FileAlreadyExistsException;
085import java.util.LinkedList;
086
087public class DiskFile extends BaseFile {
088
089    protected static Integer MAX_POOL_SIZE = 100;
090        private static final Map<String, Long> QUEUE_PATH_MAPPING = new ConcurrentHashMap<String, Long>();
091
092        private static ExecutorService splitPool = null;
093
094    protected boolean copyStructureOnly = false;
095        private FileChannel channel = null;
096        private FileLock lock = null;
097        private long modifyTimeMs = 0L;
098
099        public DiskFile(String path) {
100                super(path);
101        }
102
103        public static synchronized void initPool(int poolSize) {
104                if (splitPool == null) {
105                    MAX_POOL_SIZE = poolSize;
106                        LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
107                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
108                }
109        }
110        
111        private static synchronized void initDefaultPool(){
112                if (splitPool == null) {
113                    LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
114                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
115                }
116        }
117
118        public void split(int partSize, FilePart filePart) throws IOException {
119                if (isLink()) {
120                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
121                }
122                if (isDir()) {
123                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
124                }
125                if (!exists()) {
126                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
127                }
128                initDefaultPool();
129                splitPool.execute(new Runnable() {
130                        @Override
131                        public void run() {
132                                SeekableByteChannel ch = null;
133                                int partIndex = -1;
134                                long fileSize = 0L;
135                                try {
136                                        Path originPath = Paths.get(origin.toURI());
137                                        Thread.currentThread().setName(String.format("DiskFile-split-%s", origin.getName()));
138                                        ch = Files.newByteChannel(originPath, StandardOpenOption.READ);
139                                        ByteBuffer bf = ByteBuffer.allocate(partSize);
140                                        while (ch.read(bf) != -1) {
141                                                bf.flip();
142                                                partIndex += 1;
143                                                List<Byte> parts = new ArrayList<Byte>();
144                                                while (bf.hasRemaining()) {
145                                                        parts.add(bf.get());
146                                                }
147                                                int partsSize = parts.size();
148                                                fileSize += partsSize;
149                                                byte[] part = new byte[partsSize];
150                                                for (int i = 0; i < partsSize; i++) {
151                                                        part[i] = parts.get(i);
152                                                }
153                                                filePart.process(partIndex, part);
154                                                bf.clear();
155                                        }
156                                        filePart.completed(partIndex, getModifyTimeForClock(), fileSize);
157                                } catch (Exception e) {
158                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
159                                } finally {
160                                        if (ch != null) {
161                                                try {
162                                                        ch.close();
163                                                } catch (IOException e) {
164                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
165                                                }
166                                        }
167                                        filePart.ended(partIndex, fileSize);
168                                }
169                        }
170                });
171        }
172
173        @Override
174        public void copyTo(String toPath) throws IOException {
175                Path originPath = Paths.get(origin.toURI());
176                Path destPath = Paths.get(toPath);
177                LoggerFactory.getLogger(DiskFile.class).debug("CopyTo: {} -> {}", originPath, destPath);
178                Files.copy(originPath, destPath);
179        }
180
181        @Override
182        public void moveTo(String toPath) throws IOException {
183                Path originPath = Paths.get(origin.toURI());
184                Path destPath = Paths.get(toPath);
185                LoggerFactory.getLogger(DiskFile.class).debug("MoveTo: {} -> {}", originPath, destPath);
186                if (Files.exists(destPath)) {
187                        Files.delete(destPath);
188                }
189                Files.move(originPath, destPath);
190        }
191
192        public RemoteFile writeToRemote(RemoteFile remoteFile) throws IOException {
193                return writeToRemote(remoteFile, new Runnable() {
194                        @Override
195                        public void run() {
196
197                        }
198                });
199        }
200
201        public RemoteFile writeToRemote(RemoteFile remoteFile, Runnable completedCallback) throws IOException {
202            if(remoteFile.exists(true)){
203                throw new IOException(String.format("The remote file '%s' already exists.", remoteFile.getPath()));
204                } else {
205                        remoteFile.copyFrom(copyStructureOnly, this);
206                        if (isDir()) {
207                                startFullAsync(0, this.getPath() + "/", remoteFile.getPath() + "/", completedCallback, null);
208                        }
209                }
210                return remoteFile;
211        }
212
213        protected boolean isLogicModify() {
214                boolean b = false;
215                try {
216                        String tmpPath = toModifyTmpFilePath();
217
218                        if (tmpPath == null)
219                                return false;
220
221                        DiskFile df = null;
222                        Path logicPath = Paths.get(tmpPath);
223                        if (Files.exists(logicPath)) {
224                                df = new DiskFile(tmpPath);
225                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
226                                Timestamp dfModifyTime = df.getModifyTimeForClock();
227
228                                if (dfModifyTime == null)
229                                        return false;
230
231                                long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
232                                
233                                b = diffMs < LOGIC_TIMEOUT_MS;
234                        }
235                } catch (Exception e) {
236                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
237                }
238                return b;
239        }
240
241        protected DiskFile logicModify() {
242                return logicModify(null);
243        }
244
245        protected DiskFile logicModify(String msg) {
246                try {
247                        if (getParentFile().exists()) {
248                                String tmpLogicPath = toModifyTmpFilePath();
249
250                                if (tmpLogicPath == null)
251                                        return null;
252
253                                Path logicPath = Paths.get(tmpLogicPath);
254                                DiskFile df = new DiskFile(tmpLogicPath);
255                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
256
257                                if (msg == null) {
258                                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
259                                                        false);
260                                } else {
261                                        df.write(msg, false);
262                                }
263
264                                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
265                                return df;
266                        }
267                } catch (Exception e) {
268                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
269                }
270                return null;
271        }
272
273        protected void removeLogicModify() {
274                try {
275                        String tmpPath = toModifyTmpFilePath();
276                        if (tmpPath != null) {
277                                DiskFile df = new DiskFile(tmpPath);
278                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
279                                if (df.exists()) {
280                                        df.delete();
281                                }
282                        }
283                } catch (Exception e) {
284                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
285                }
286        }
287
288        protected boolean isLogicAccess() {
289                boolean b = false;
290                try {
291                        String tmpPath = toAccessTmpFilePath();
292
293                        if (tmpPath == null)
294                                return false;
295
296                        DiskFile df = null;
297                        Path logicPath = Paths.get(tmpPath);
298                        if (Files.exists(logicPath)) {
299                                df = new DiskFile(tmpPath);
300                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
301                                Timestamp dfModifyTime = df.getModifyTimeForClock();
302                                
303                                long ltms = df.isFile() ? LOGIC_TIMEOUT_MS*2 : LOGIC_TIMEOUT_MS;
304
305                                if (dfModifyTime == null)
306                                        return false;
307
308                                long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
309                                
310                                b = diffMs < ltms;
311                        }
312                } catch (Exception e) {
313                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
314                }
315                return b;
316        }
317
318        protected DiskFile logicAccess() {
319                return logicAccess(null);
320        }
321
322        protected DiskFile logicAccess(String msg) {
323                try {
324                        if (getParentFile().exists()) {
325                                String tmpLogicPath = toAccessTmpFilePath();
326                                if (tmpLogicPath == null)
327                                        return null;
328
329                                Path logicPath = Paths.get(tmpLogicPath);
330                                DiskFile df = new DiskFile(tmpLogicPath);
331                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
332                                if (msg == null) {
333                                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
334                                                        false);
335                                } else {
336                                        df.write(msg, false);
337                                }
338
339                                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
340                                return df;
341                        }
342                } catch (Exception e) {
343                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
344                }
345                return null;
346        }
347
348        protected void removeLogicAccess() {
349                try {
350                        String tmpPath = toAccessTmpFilePath();
351                        if (tmpPath != null) {
352                                DiskFile df = new DiskFile(tmpPath);
353                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
354                                if (df.exists()) {
355                                        df.delete();
356                                }
357                        }
358                } catch (Exception e) {
359                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
360                }
361        }
362        
363        protected boolean isLogicCheck() {
364                boolean b = false;
365                try {
366                        String tmpPath = toCheckTmpFilePath();
367
368                        if (tmpPath == null)
369                                return false;
370
371                        DiskFile df = null;
372                        Path logicPath = Paths.get(tmpPath);
373                        if (Files.exists(logicPath)) {
374                                df = new DiskFile(tmpPath);
375                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
376                                Timestamp dfModifyTime = df.getModifyTimeForClock();
377
378                                if (dfModifyTime == null)
379                                        return false;
380
381                                long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
382                                
383                                b = diffMs < LOGIC_CHECK_TIMEOUT_MS;
384                        }
385                } catch (Exception e) {
386                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
387                }
388                return b;
389        }
390
391        protected DiskFile logicCheck() {
392                return logicCheck(null);
393        }
394
395        public DiskFile logicCheck(String msg) {
396                try {
397                        if (getParentFile().exists()) {
398                                String tmpLogicPath = toCheckTmpFilePath();
399                                if (tmpLogicPath == null)
400                                        return null;
401
402                                Path logicPath = Paths.get(tmpLogicPath);
403                                DiskFile df = new DiskFile(tmpLogicPath);
404                                df.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
405                                if (msg == null) {
406                                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
407                                                        false);
408                                } else {
409                                        df.write(msg, false);
410                                }
411
412                                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
413                                return df;
414                        }
415                } catch (Exception e) {
416                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
417                }
418                return null;
419        }    
420
421        @Override
422        public boolean beforeDelete(boolean realDeleted) {
423                return true;
424        }
425        
426        @Override
427        public void afterDelete(boolean realDeleted) {
428            
429        }       
430        
431        @Override
432        public boolean beforeMkdirs() {
433            return true;
434        }
435        
436        @Override
437        public void afterMkdirs() {
438            
439        }
440        
441        @Override
442        public boolean beforeCreateLink(String target) {
443            return true;
444        }
445        
446        @Override
447        public void afterCreateLink(String target) {
448            
449        }
450        
451        @Override
452        public boolean beforeWrite(){
453            return true;
454        }
455        
456        @Override
457        public void afterWrite(){
458            
459        }
460
461        @Override
462        public boolean delete() throws IOException {
463                return deleteFrom();
464        }
465
466        private boolean deleteFrom() throws IOException {
467                boolean allowed = beforeDelete(true);
468                if (allowed) {
469                        LoggerFactory.getLogger(DiskFile.class).debug("Delete: {}", getPath());
470                        if (isFile()) {
471                                Path originPath = Paths.get(origin.toURI());
472                                boolean exists = Files.exists(originPath);
473                                if (exists) {
474                                        Files.deleteIfExists(Paths.get(origin.toURI()));
475                                        afterDelete(true);
476                                        return true;
477                                }
478                        }
479                        if (isDir()) {
480                                Path originPath = Paths.get(origin.toURI());
481                                boolean exists = Files.exists(originPath);
482                                if (exists) {
483                                        Files.walk(originPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
484                                        afterDelete(true);
485                                        return true;
486                                }
487                        }
488                        if (isLink()) {
489                                Path originPath = Paths.get(origin.toURI());
490                                boolean exists = Files.exists(originPath);
491                                if (exists) {
492                                        Files.deleteIfExists(Paths.get(origin.toURI()));
493                                        afterDelete(true);
494                                        return true;
495                                }
496                        }
497                }
498                return false;
499        }
500
501        @Override
502        public boolean isFile() throws IOException {
503                return this.origin.isFile() && !isLink();
504        }
505
506        @Override
507        public boolean isDirectory() throws IOException {
508                return this.origin.isDirectory() && !isLink();
509        }
510
511        @Override
512        public boolean isLink() throws IOException {
513                return Files.isSymbolicLink(Paths.get(this.origin.toURI()));
514        }
515
516        @Override
517        public boolean exists() throws IOException {
518                return this.origin.exists();
519        }
520
521        @Override
522        public boolean mkdirs() throws IOException {
523            boolean allowed = beforeMkdirs();
524            if(allowed){
525                LoggerFactory.getLogger(DiskFile.class).debug("Mkidrs: {}", getPath());
526                boolean b = this.origin.mkdirs();
527                if (b && modifyTimeMs > 0) {
528                        setModifyTime(new Timestamp(modifyTimeMs));
529                }
530                afterMkdirs();
531                return b;
532            }
533            return false;
534        }
535
536    @Override
537        public boolean write(byte[] data, boolean append) throws IOException {
538            
539            boolean allowed = beforeWrite();
540            
541            if(!allowed) return false;
542
543                if (data == null)
544                        data = new byte[] {};
545
546                try {
547
548                        OpenOption[] options = null;
549                        if (append) {
550                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
551                                                StandardOpenOption.APPEND };
552                        } else {
553                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
554                                                StandardOpenOption.TRUNCATE_EXISTING };
555                        }
556
557                        File parent = origin.getParentFile();
558                        if (!parent.exists()) {
559                                parent.mkdirs();
560                        }
561
562                        Path path = Paths.get(origin.toURI());
563                        channel = FileChannel.open(path, options);
564                        lock = channel.lock();
565                        if (lock != null) {
566                                ByteBuffer buf = ByteBuffer.wrap(data);
567                                channel.write(buf);
568                                if (modifyTimeMs > 0) {
569                                        setModifyTime(new Timestamp(modifyTimeMs));
570                                }
571                        }
572                        return true;
573                } catch (IOException e) {
574                        throw e;
575                } finally {
576                        close();
577                }
578        }
579
580        public boolean manualOpen(boolean append) throws IOException, FileAlreadyExistsException {
581
582            boolean allowed = beforeWrite();
583            
584            if(!allowed) return false;
585            
586                OpenOption[] options = null;
587                if (append) {
588                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
589                                        StandardOpenOption.APPEND };
590                } else {
591                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
592                                        StandardOpenOption.TRUNCATE_EXISTING };
593                }
594                Path path = Paths.get(origin.toURI());
595                File parent = origin.getParentFile();
596                if (parent != null && !parent.exists() && !parent.mkdirs()) {
597                        throw new IOException(String.format("Failed to create parent directory: %s", parent));
598                }
599                if (!Files.exists(path)) {
600                        try {
601                                Files.createFile(path);
602                        } catch (FileAlreadyExistsException e) {
603                                throw e;
604                        }
605                }
606                channel = FileChannel.open(path, options);
607                lock = channel.lock();
608                
609                return true;
610        }
611
612        public void manualWrite(byte[] data) throws IOException {
613                if (data == null)
614                        data = new byte[] {};
615
616                if (lock != null) {
617                        ByteBuffer buf = ByteBuffer.wrap(data);
618                        channel.write(buf);
619                        if (modifyTimeMs > 0) {
620                                setModifyTime(new Timestamp(modifyTimeMs));
621                        }
622                }
623        }
624        
625        public void manualWrite(String data) throws IOException {
626                 manualWrite(data.getBytes(CHARSET));
627        }       
628
629        public void manualClose() throws IOException {
630                close();
631        }
632
633        @Override
634        public boolean write(byte[] data) throws IOException {
635                return write(data, false);
636        }
637
638        @Override
639        public boolean write(String data, boolean append) throws IOException {
640                return write(data.getBytes(CHARSET), append);
641        }
642
643        public boolean write(String data, String charset,boolean append) throws IOException {
644                return write(data.getBytes(charset), append);
645        }
646        
647        @Override
648        public boolean write(String data) throws IOException {
649                return write(data.getBytes(CHARSET));
650        }
651        
652        public boolean write(String data,String charset) throws IOException {
653                return write(data.getBytes(charset));
654        }       
655
656        @Override
657        public boolean createLink(String target) throws IOException {
658            boolean allowed = beforeCreateLink(target);
659            if(allowed){
660                LoggerFactory.getLogger(DiskFile.class).debug("CreateLink: {}", getPath());
661                try {
662                        Path link = Paths.get(origin.toURI());
663                        Path targetPath = Paths.get(target);
664                        Files.createSymbolicLink(link, targetPath);
665                        if (modifyTimeMs > 0) {
666                                setModifyTime(new Timestamp(modifyTimeMs));
667                        }
668                        afterCreateLink(target);
669                        return true;
670                } catch (IOException e) {
671                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
672                }
673            }
674                return false;
675        }
676
677        @Override
678        public String readLink() throws IOException {
679                Path link = Paths.get(origin.toURI());
680                return Files.readSymbolicLink(link).toString();
681        }
682
683        @Override
684        public byte[] readAllBytes() throws IOException {
685                return readAllBytesFrom();
686        }
687        
688        private byte[] readAllBytesFrom() throws IOException {
689                if (isLink()) {
690                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
691                }
692                if (isDir()) {
693                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
694                }
695                if (!exists()) {
696                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
697                }
698                return Files.readAllBytes(Paths.get(getPath()));
699        }
700
701        @Override
702        public String readAllString() throws IOException {
703                return readAllString(BaseFile.CHARSET);
704        }
705        
706        public String readAllString(String charset) throws IOException {
707                ByteArrayOutputStream baos = null;
708                try {
709                        byte[] bytes = readAllBytes();
710                        baos = new ByteArrayOutputStream();
711                        baos.write(bytes);
712                        baos.flush();
713                        return baos.toString(charset);
714                } finally {
715                        if (baos != null) {
716                                try {
717                                        baos.close();
718                                } catch (IOException e) {
719                                        Logger.systemError(DiskFile.class, e.getMessage(), e);
720                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
721                                }
722                        }
723                }
724        }       
725
726        @Override
727        public long size() throws IOException {
728                if (isFile()) {
729                        Path path = Paths.get(origin.getAbsolutePath());
730                        return Files.size(path);
731                }
732                return 0L;
733        }
734
735        private void close() {
736                if (lock != null) {
737                        try {
738                                lock.release();
739                        } catch (IOException e) {
740                                Logger.systemError(DiskFile.class, e.getMessage(), e);
741                        }
742                        try {
743                                lock.close();
744                        } catch (IOException e) {
745                                Logger.systemError(DiskFile.class, e.getMessage(), e);
746                        } finally {
747                                lock = null;
748                        }
749                }
750
751                if (channel != null) {
752                        try {
753                                channel.close();
754                        } catch (IOException e) {
755                                Logger.systemError(DiskFile.class, e.getMessage(), e);
756                        } finally {
757                                channel = null;
758                        }
759                }
760                afterWrite();
761        }
762
763        @Override
764        public boolean complete() throws IOException {
765                close();
766                return true;
767        }
768
769        public void setModifyTimeMs(long modifyTimeMs) throws IOException {
770                this.modifyTimeMs = modifyTimeMs;
771        }
772
773        public void setModifyTimeMsFromClock(long modifyTimeMsFromClock) throws IOException {
774                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
775                String localTzId = Calendar.getInstance().getTimeZone().getID();
776                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, new Timestamp(modifyTimeMsFromClock));
777                this.modifyTimeMs = fileLocalTimestamp.getTime();
778        }
779
780        @Override
781        public void setModifyTime(Timestamp modifyTime) throws IOException {
782                if (exists()) {
783                        Path originPath = Paths.get(origin.toURI());
784                        Files.setLastModifiedTime(originPath, FileTime.fromMillis(modifyTime.getTime()));
785                }
786        }
787
788        public void setModifyTimeFromClock(Timestamp modifyTimeFromClock) throws IOException {
789                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
790                String localTzId = Calendar.getInstance().getTimeZone().getID();
791                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, modifyTimeFromClock);
792                setModifyTime(fileLocalTimestamp);
793        }
794
795        @Override
796        public Timestamp getModifyTime() throws IOException {
797                Path originPath = Paths.get(origin.toURI());
798                if (Files.exists(originPath)) {
799                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
800                        FileTime fileTime = attr.lastModifiedTime();
801                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
802                        return fileLocalTimestamp;
803                }
804                return null;
805        }
806
807        public Timestamp getModifyTimeForClock() throws IOException {
808                Path originPath = Paths.get(origin.toURI());
809                if (Files.exists(originPath)) {
810                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
811                        FileTime fileTime = attr.lastModifiedTime();
812                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
813                        String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
814                        String localTzId = Calendar.getInstance().getTimeZone().getID();
815                        Timestamp fileRemoteTimestamp = timeZoneConver(remoteTzId, localTzId, fileLocalTimestamp);
816                        return fileRemoteTimestamp;
817                }
818                return null;
819        }
820
821        public void startFullAsync(long timerMs, String replaceTo) throws IOException {
822                startFullAsync(timerMs, getPath() + "/", replaceTo, null, null);
823        }
824
825        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback) throws IOException {
826                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, null);
827        }
828
829        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback,
830                        Runnable checkDeleteCompletedCallback) throws IOException {
831                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, checkDeleteCompletedCallback);
832        }
833
834        private void startFullAsync(long timerMs,String replaceRootPath, String replaceTo,
835                        Runnable completedCallback, Runnable checkDeleteCompletedCallback) throws IOException {
836                            
837                if(!RemoteFile.FIRST_LOADED){
838                    RemoteFile firstLoaded = new RemoteFile(replaceTo);
839                }           
840                
841                Runnable runnable = new Runnable() {
842                        @Override
843                        public void run() {
844                                try {
845                                        startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
846                                } catch (Exception e) {
847                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
848                                }
849                        }
850                };
851                Executors.newFixedThreadPool(1).execute(runnable);
852
853                runnable = new Runnable() {
854                        @Override
855                        public void run() {
856                                try {
857                                        startFullSyncForDelete(-1,timerMs,replaceTo, replaceRootPath,
858                                                        checkDeleteCompletedCallback);
859                                } catch (Exception e) {
860                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
861                                }
862                        }
863                };
864                Executors.newFixedThreadPool(1).execute(runnable);
865        }
866        
867        private synchronized void startFullSyncForDelete(final Integer filePartDataTable,final long timerMs,final String replaceTo,
868                        final String replaceRootPath, final Runnable completedCallback) throws IOException {
869                DiskFile the = this;
870                RemoteFile rootDir = null;
871                if (!CommonTools.isBlank(replaceTo)) {
872                                rootDir = new RemoteFile(replaceTo);
873                }
874                if (rootDir == null) {
875                        throw new IOException("The root folder is null.");
876                }
877                RemoteFile _rootDir = rootDir;
878        Integer maxDataTable = rootDir.getMaxFilePartDataTable();
879        
880        Integer _filePartDataTable = filePartDataTable;
881        
882        if(filePartDataTable > maxDataTable) _filePartDataTable = -1;
883        
884        final Integer finalFilePartDataTable = _filePartDataTable;
885        
886            final CacheArray rows = new CacheArray();
887            
888                rows.filter(getScanDeleteFilter(rootDir,finalFilePartDataTable,timerMs,replaceTo,replaceRootPath,completedCallback));
889
890        Runnable runnable = new Runnable(){
891            @Override
892            public void run(){
893                try{
894                        _rootDir.listAllForScanDelete(finalFilePartDataTable,rows);
895                }catch(Exception e){
896                    LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
897                }
898            }
899        };
900        Executors.newFixedThreadPool(1).execute(runnable);
901        }       
902        
903        private CacheArrayFilter getScanDeleteFilter(final RemoteFile rootDir,final Integer filePartDataTable,final long timerMs,final String replaceTo,
904                        final String replaceRootPath, final Runnable completedCallback){
905                DiskFile the = this;
906                CacheArrayFilter filter = new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
907                        @Override
908                        public void execute(Integer index, Object o) {
909                
910                                try {
911                                        Map<String, Object> item = (Map<String, Object>) o;
912                                        Object id = item.get("id");
913                                        String parent = (String) item.get("file_parent_path");
914                                        String fileName = (String) item.get("file_name");
915                                        String pf = String.format("%s/%s", parent, fileName);
916                                        RemoteFile rf = new RemoteFile(pf);
917                                        Thread.currentThread().setName(String.format("DiskFile-startFullSyncForDelete-(DataTable=%s Index=%s)-%s",filePartDataTable,index,fileName));
918
919                                        if (memoryUsage() < RemoteFile.MAX_MEMORY_USAGE) {
920                                        String toDf = pf;
921                                        if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
922                                                toDf = pf.replaceFirst(replaceTo + "[/]*", replaceRootPath + "/");
923                                        }
924                                        
925                                        DiskFile df = new DiskFile(toDf);
926                                        df.copyAttrs(df, df.copyStructureOnly, df.syncRoot);
927                                        boolean isDebug = isDebug(rf);
928                                        boolean isRoot = rootDir.getPath().equals(rf.getPath());
929                                        boolean isSyncedHost = rf.isSyncedOnHostname(id);
930                                        boolean rfExists = rf.exists();
931    
932                                        boolean allowedForceDelete = rootDir.exists() && !isRoot && isSyncedHost && rfExists
933                                                        && !df.exists() && !df.isLogicModify();
934                                                        
935                                        if (allowedForceDelete) {
936                                                df.logicModify();
937                                                if (isDebug) {
938                                                        LoggerFactory.getLogger(DiskFile.class).mark("ForceRemoteDelete - {}", rf.getPath());
939                                                }
940                                                rf.forceDeleteAll(true);
941                                                df.removeLogicModify();
942                                        }
943                                        }else{
944                                            LoggerFactory.getLogger(DiskFile.class).warn("Over max memory usage 'MAX_MEMORY_USAGE={}'.",RemoteFile.MAX_MEMORY_USAGE);
945                                        }
946                                } catch (Exception e) {
947                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
948                                }
949                        }
950
951                        @Override
952                        public void completed(Integer size) {
953                                if (isStopSync()) {
954                                        LoggerFactory.getLogger(DiskFile.class).mark("Stoped start full sync for delete.");
955                                }
956                                if (timerMs <= 0) {
957                                        LoggerFactory.getLogger(DiskFile.class).mark("Completed start full sync for delete.");
958                                }
959
960                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
961                                
962                                if (isRootCompleted && completedCallback != null) {
963                                        completedCallback.run();
964                                }
965                                
966                                if (!isStopSync() && timerMs > 0) {
967                                    while(true){
968                                        
969                                        if(isStopSync()) break;
970                                        
971                                        try {
972                                                Thread.sleep(timerMs);
973                                        } catch (InterruptedException e) {
974                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
975                                        }
976                                        boolean allowed = checkUsage(rootDir);
977                                        if(allowed){
978                                                try {
979                                                    if(!isStopSync()){
980                                                        startFullSyncForDelete(filePartDataTable + 1,timerMs, replaceTo, replaceRootPath,
981                                                                        completedCallback);
982                                                    }
983                                                } catch (IOException e) {
984                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
985                                                }
986                                                break;
987                                        }
988                                    }
989                                }
990                        }
991                };
992                return filter;
993        }
994
995        private void startFullAsyncThread(final long timerMs, String replaceRootPath,
996                        String replaceTo, final Runnable completedCallback) {
997                stopSync = false;
998                this.setCopyStructureOnly(RemoteFile.COPY_STRUCTURE_ONLY);
999                final DiskFile the = this;
1000                Thread.currentThread()
1001                                                .setName(String.format("DiskFile-startFullAsyncThread-%s",the.getOrigin().getName()));
1002                replaceRootPath = replacePath(replaceRootPath);
1003                replaceTo = replacePath(replaceTo);
1004                DiskFile rootDisk = new DiskFile(replaceRootPath);
1005                boolean rootExists = false;
1006                try{
1007                    rootExists = rootDisk.exists();
1008                }catch(Exception e){
1009                    LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(),e);
1010                }
1011                if(!rootExists){
1012                    Logger.systemError(DiskFile.class,"The root path '{}' does not exist.",rootDisk.getPath());
1013                }
1014        boolean isRoot = the.getPath().equals(rootDisk.getPath());
1015        LoggerFactory.getLogger(DiskFile.class).debug("IsRoot: {} - {}",isRoot,the.getPath());
1016        LoggerFactory.getLogger(DiskFile.class).debug("CacheArrayUsage: {}",CacheArray.getUsage());
1017                if (rootExists && (isRoot || !the.isLogicAccess())) {
1018                        if(!isRoot) the.logicAccess();
1019                        
1020                        DirectoryStream<Path> stream = null;
1021                        try {
1022                                if (syncRoot == null) {
1023                                        File replaceRootFile = new File(replaceRootPath);
1024                                        if (replaceRootFile.getParent() == null) {
1025                                                throw new IOException("Cannot sync the root path.");
1026                                        }
1027                                        String replaceRootParent = replacePath(replaceRootFile.getParent());
1028                                        syncRoot = new File(String.format("%s/%s", replaceRootParent, replaceRootFile.getName()));
1029                                }
1030
1031                                if (!CommonTools.isBlank(replaceTo)) {
1032                                        RemoteFile rootDir = new RemoteFile(replaceTo);
1033                                        if (!rootDir.exists()) {
1034                                                rootDir.setModifyTime(this.getModifyTimeForClock());
1035                                                rootDir.mkdirs();
1036                                        }
1037                                }
1038                                Path originPath = Paths.get(origin.toURI());
1039                                CacheArray rows = new CacheArray();
1040                                CacheArrayFilter filter = getFullAsyncFilter(the, timerMs, replaceRootPath, replaceTo,
1041                                                completedCallback);
1042                                rows.filter(filter);
1043                                stream = Files.newDirectoryStream(originPath);
1044                                for (Path p : stream) {
1045                                        if (p != null) {
1046                                            File f = p.toFile();
1047                                            boolean isFile = f.isFile();
1048                                            if(isFile){
1049                                                boolean isWritingDf = f.getName().matches(TMP_WRITING_SWP);
1050                                                if(isWritingDf){
1051                                BasicFileAttributes attributes = Files.readAttributes(p, BasicFileAttributes.class);
1052                                FileTime creationTime = attributes.creationTime();
1053                                boolean timeouted = (Calendar.getInstance().getTimeInMillis() - creationTime.toMillis()) > (RemoteFile.BATCH_SIZE*LOGIC_TIMEOUT_MS);
1054                                if(timeouted){
1055                                    try{
1056                                        boolean exists = Files.exists(p);
1057                                                        if (exists) {
1058                                                                Files.deleteIfExists(p);
1059                                                        }
1060                                    }catch(Exception e){
1061                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1062                                    }
1063                                }
1064                                                }
1065                                            }
1066                                                boolean allowedPath = the.checkSyncPathAvailable(p.toString());
1067                                                if(!allowedPath){
1068                                                    LoggerFactory.getLogger(DiskFile.class).debug("Ignored: {}",p.toString());
1069                                                }
1070                                                if (allowedPath) {
1071                                                        rows.add(p);
1072                                                }
1073                                        }
1074                                }
1075                                rows.add(null);
1076                        } catch (Exception e) {
1077                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1078                                removeLogicAccess();
1079                                if (!(e instanceof NoSuchFileException)) {
1080                                        if (!isStopSync() && timerMs > 0) {
1081                                                try {
1082                                                        Thread.sleep(timerMs);
1083                                                } catch (InterruptedException ee) {
1084                                                        LoggerFactory.getLogger(DiskFile.class).error(ee.getMessage(), ee);
1085                                                }
1086                                                if(!isStopSync()){
1087                                                    startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
1088                                                }
1089                                        }
1090                                }
1091                        } finally {
1092                                if (stream != null) {
1093                                        try {
1094                                                stream.close();
1095                                        } catch (IOException e) {
1096                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1097                                        }
1098                                }
1099                        }
1100                }
1101        }
1102
1103        private CacheArrayFilter getFullAsyncFilter(DiskFile the, long timerMs,
1104                        String replaceRootPath, String replaceTo, Runnable completedCallback) {
1105
1106                return new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
1107                        RemoteFile rf = null;
1108            final CacheArrayFilter theFilter = this;    
1109                        @Override
1110                        public void completed(Integer size) {
1111                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1112                                
1113                                if(!isRootCompleted) the.removeLogicAccess();
1114                                
1115                                if (isRootCompleted) {
1116                                        try {
1117                        if (rf != null && the.isStopSync() && isDebug(rf)) {
1118                                                LoggerFactory.getLogger(DiskFile.class).mark("StopedRootScan - {}", the.getPath());
1119                                        }                                                               
1120                                                if (!the.isStopSync() && timerMs > 0) {
1121                                                        Thread.sleep(timerMs);
1122                                                if (!the.isStopSync()) {
1123                                                if (rf != null && isDebug(rf)) {
1124                                                        LoggerFactory.getLogger(DiskFile.class).mark("RootScaning - {}", the.getPath());
1125                                                }                                                   
1126                                                            the.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1127                                                                            completedCallback);
1128                                                }
1129                                                }
1130                                                if (completedCallback != null) {
1131                                                        completedCallback.run();
1132                                                }
1133                                        } catch (Exception e) {
1134                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1135                                        }
1136                                }
1137                        }
1138
1139                        @Override
1140                        public void execute(Integer index, Object o) {
1141                DiskFile df = null;
1142                try{
1143                                Path p = (Path) o;
1144                                String pf = replacePath(p.toFile().getAbsolutePath());
1145                                Thread.currentThread()
1146                                                .setName(String.format("DiskFile-getFullAsyncFilter-%s", p.toFile().getParentFile().getName()));
1147                                String toRf = pf;
1148                                if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
1149                                        toRf = pf.replaceFirst(replaceRootPath + "[/]*", replaceTo + "/");
1150                                }
1151    
1152                                df = new DiskFile(pf);
1153                                df.copyAttrs(df, the.copyStructureOnly, the.syncRoot);
1154                                rf = new RemoteFile(toRf);
1155    
1156                    boolean isRoot = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1157                    
1158                    boolean allowedConn = checkUsage(rf);
1159                                        String fileName = p.toFile().getName();
1160                                        boolean isDebug = isDebug(rf);
1161
1162                                        boolean timeouted = rf.isTimeout();
1163                                        if (timeouted && RemoteFile.addQueuePathMapping(rf.getPath()) && allowedConn) {
1164                                                if (isDebug) {
1165                                                        LoggerFactory.getLogger(DiskFile.class).mark("ForceDelete(Timeouted) - {}",
1166                                                                        rf.getPath());
1167                                                }                                                           
1168                                                rf.forceDeleteFile(false);
1169                                                RemoteFile.removeQueuePathMapping(rf.getPath());
1170                                        }
1171                                        Timestamp modifyTime = df.getModifyTimeForClock();
1172                                        boolean allowedModifyTime = modifyTime != null;
1173                                        boolean allowed = DiskFile.checkQueuePathMapping() && !isStopSync() && allowedModifyTime && allowedConn;
1174                                        
1175                                        if(isDebug){
1176                                            debugLog("CheckUsage",rf,this);
1177                                        }
1178                                        
1179                                        if(!isRoot && allowed){
1180                                            allowed = !df.isLogicCheck();
1181                                        }
1182                                        
1183                                        if (allowed) {
1184                                            
1185                                            if(!isRoot) df.logicCheck();
1186                                            
1187                                                rf.setModifyTime(modifyTime);
1188                                                long dfModifyTimeMs = modifyTime.getTime();
1189                                                boolean blocked = false;
1190                                                boolean isRealDelete = rf.isLastOperateRealDelete();
1191
1192                                                if (isRealDelete) {
1193                                                        String clientHostname = CommonTools.getHostname();
1194                                                        String delHostname = rf.getLastSourceHostname();
1195                                                        boolean isOwnerDel = clientHostname.equals(delHostname);
1196                                                        if (isOwnerDel) {
1197                                                                Timestamp deletedTime = rf.getLastOperateTime();
1198                                                                if (deletedTime != null) {
1199                                                                        boolean waiting = new Clock().getTime() - deletedTime.getTime() <= LOGIC_TIMEOUT_MS;
1200                                                                        if (waiting) {
1201                                                                                blocked = true;
1202                                                                        }
1203                                                                }
1204                                                        } else {
1205                                                                Timestamp deletedTime = rf.getLastOperateTime();
1206                                                                if (deletedTime != null) {
1207                                                                        boolean olded = modifyTime.getTime() < deletedTime.getTime();
1208                                                                        if (olded) {
1209                                                                                blocked = true;
1210                                                                        }
1211                                                                }
1212                                                        }
1213                                                }
1214
1215                                                boolean haveParent = rf.getParentFile().exists();
1216                                                if (!blocked && haveParent) {    
1217                                                        if (df.isLink()) {
1218                                                                df.logicAccess();
1219                                                                if (rf.exists()) {
1220                                                                        if (rf.isLink()) {
1221                                                                                long rfModifyTimeMs = rf.getModifyTime().getTime();
1222                                                                                boolean changedTime = dfModifyTimeMs > rfModifyTimeMs;
1223                                                                                boolean changedValue = !rf.readLink().equals(df.readLink());
1224                                                                                boolean changed = changedTime && changedValue;
1225                                                                                if (changed) {
1226                                                                                RemoteFile prf = rf.getParentFile();
1227                                                                                if(prf != null && prf.exists()){                                                                                    
1228                                                                                        if (isDebug) {
1229                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1230                                                                                                                rf.getPath());
1231                                                                                        }
1232                                                                                        rf.forceDeleteLink(false);
1233                                                                                        rf.createLink(df.readLink());
1234                                                                                }
1235                                                                                }
1236                                                                        } else {
1237                                                                        RemoteFile prf = rf.getParentFile();
1238                                                                        if(prf != null && prf.exists()){                                                                            
1239                                                                                if (isDebug) {
1240                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1241                                                                                                        rf.getPath());
1242                                                                                }
1243                                                                                rf.forceDeleteLink(false);
1244                                                                                rf.createLink(df.readLink());
1245                                                                        }
1246                                                                        }
1247                                                                } else {
1248                                                                        RemoteFile prf = rf.getParentFile();
1249                                                                        if(prf != null && prf.exists()){
1250                                                                        if (isDebug) {
1251                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}", rf.getPath());
1252                                                                        }       
1253                                                                            rf.createLink(df.readLink());
1254                                                                        }
1255                                                                }
1256                                                                df.removeLogicAccess();
1257                                                        } else if (df.isFile()) {
1258                                                                if (!df.isLogicModify() && !df.isLogicAccess()) {
1259                                                                        if (rf.exists()) {
1260                                                                                if (rf.isFile()) {
1261                                                                                        long rfModifyTimeMs = rf.getModifyTime().getTime();
1262                                                                                        if (dfModifyTimeMs > rfModifyTimeMs) {
1263                                                                                            RemoteFile prf = rf.getParentFile();
1264                                                                                            if(prf != null && prf.exists()){
1265                                                                                                if (DiskFile.addQueuePathMapping(df.getPath())) {
1266                                                                                                        if (isDebug) {
1267                                                                                                                LoggerFactory.getLogger(DiskFile.class)
1268                                                                                                                                .mark("RemoteQueuing - {}", df.getPath());
1269                                                                                                        }
1270                                                                                                        df.logicAccess();
1271                                                                                                        rf.forceDeleteFile(false);
1272                                                                                                        df.writeToRemote(rf);
1273                                                                                                }
1274                                                                                            }
1275                                                                                        }
1276                                                                                } else {
1277                                                                                        rf.forceDeleteFile(false);
1278                                                                                }
1279                                                                        } else {
1280                                                                            RemoteFile prf = rf.getParentFile();
1281                                                                            if(prf != null && prf.exists()){
1282                                                                                if (DiskFile.addQueuePathMapping(df.getPath())) {
1283                                                                                        if (isDebug) {
1284                                                                                                LoggerFactory.getLogger(DiskFile.class)
1285                                                                                                                .mark("RemoteQueuing - {}", df.getPath());
1286                                                                                        }
1287                                                                                        df.logicAccess();
1288                                                                                        df.writeToRemote(rf);
1289                                                                                }
1290                                                                            }
1291                                                                        }
1292                                                                }
1293                                                        } else if (df.isDir()) {
1294                                                                boolean isCompleted = !df.isLogicAccess();
1295                                                                if (isStopSync() && isDebug) {
1296                                                                        LoggerFactory.getLogger(DiskFile.class).mark("Stopping queue");
1297                                                                }
1298                                                                if (!isStopSync() && isCompleted) {
1299                                                                        if (isDebug) {
1300                                                                                LoggerFactory.getLogger(DiskFile.class).debug("Scaning - {}", df.getPath());
1301                                                                        }
1302                                                                        boolean allowedPoolSize = theFilter.getCacheArray().getUsingPoolSize() < theFilter.getCacheArray().getMaxPoolSize();
1303                                                                        if (rf.exists()) {
1304                                                                                if (rf.isDir()) {
1305                                                                                    allowedConn = allowedPoolSize && checkUsage(rf);
1306                                                                                    if(isRoot || allowedConn){
1307                                                                                        df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1308                                                                                                        completedCallback);
1309                                                                                    }
1310                                                                                } else {
1311                                                                                        if (rf.isDir())
1312                                                                                                rf.forceDeleteDir(false);
1313
1314                                                                                        if (rf.isFile())
1315                                                                                                rf.forceDeleteFile(false);
1316
1317                                                                                        if (rf.isLink())
1318                                                                                                rf.forceDeleteLink(false);
1319                                            
1320                                            RemoteFile prf = rf.getParentFile();
1321                                            if(isRoot || (prf != null && prf.exists())){
1322                                                                                        if (isDebug) {
1323                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("ReCreateRemoteDir - {}",
1324                                                                                                                rf.getPath());
1325                                                                                        }                                                
1326                                                                                        rf.mkdirs();
1327                                                                                        allowedConn = allowedPoolSize && checkUsage(rf);
1328                                                if(isRoot || allowedConn){
1329                                                                                                df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1330                                                                                                                completedCallback);
1331                                                }
1332                                            }
1333                                                                                }
1334                                                                        } else {
1335                                                                                RemoteFile prf = rf.getParentFile();
1336                                                                                if(isRoot || (prf != null && prf.exists())){
1337                                                                                if (isDebug) {
1338                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("CreateRemoteDir - {}",
1339                                                                                                        rf.getPath());
1340                                                                                }                                                                                   
1341                                                                                rf.mkdirs();
1342                                                                            allowedConn = allowedPoolSize && checkUsage(rf);
1343                                            if(isRoot || allowedConn){
1344                                                                                    df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1345                                                                                                    completedCallback);
1346                                            }
1347                                                                                }
1348                                                                        }
1349                                                                        if(!isRoot && !allowedPoolSize) df.removeLogicAccess();
1350                                                                }
1351                                                        }
1352                                                }
1353                                        }
1354                                } catch (Exception e) {
1355                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1356                                        if (df != null) {
1357                                                df.removeLogicAccess();
1358                                        }
1359                                }
1360                        }
1361
1362                };
1363        }
1364
1365        public void setCopyStructureOnly(boolean copyStructureOnly) {
1366                this.copyStructureOnly = copyStructureOnly;
1367        }
1368
1369        public boolean isCopyStructureOnly() {
1370                return this.copyStructureOnly;
1371        }
1372
1373        private boolean isViEditSwp(String fileName) {
1374                return fileName.matches(TMP_MATCHES_EDIT) || fileName.matches(TMP_MATCHES_VI_SWP);
1375        }
1376
1377        protected DiskFile copyAttrs(DiskFile source, boolean _copyStructureOnly, File _syncRoot) {
1378                source.setCopyStructureOnly(_copyStructureOnly);
1379                source.syncRoot = _syncRoot;
1380                return source;
1381        }
1382
1383        public File getParentFile() {
1384                return new File(getParent());
1385        }
1386
1387        protected boolean checkSyncPathAvailable(String path) {
1388            try{
1389                    return checkSyncPathAvailable(path, RemoteFile.SHOW_HIDDEN, RemoteFile.SHOW_LINK, RemoteFile.MAX_FILE_SZIE,
1390                        SYNC_PATH_ALLOWED,
1391                                SYNC_PATH_IGNORED);
1392            }catch(Exception e){
1393                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1394                return false;
1395            }
1396        }
1397
1398        protected static synchronized boolean addQueuePathMapping(String path) {
1399                Long addTimeMs = null;
1400        boolean allowed = checkQueuePathMapping();
1401        
1402        if(!allowed) return false;
1403
1404                addTimeMs = QUEUE_PATH_MAPPING.get(path);
1405
1406                if (addTimeMs == null) {
1407                        QUEUE_PATH_MAPPING.put(path, System.currentTimeMillis());
1408                        return true;
1409                }
1410
1411                return false;
1412        }
1413
1414        public static synchronized boolean checkQueuePathMapping() {
1415                Long addTimeMs = null;
1416                Map<String, Long> copyQueueMap = getQueuePathMapping();
1417                List<String> keyList = new ArrayList<String>(copyQueueMap.keySet());
1418                for (String key : keyList) {
1419                        addTimeMs = copyQueueMap.get(key);
1420                        if (addTimeMs != null) {
1421                                boolean timeout = (System.currentTimeMillis() - addTimeMs) >= LOGIC_TIMEOUT_MS;
1422
1423                                if (timeout) {
1424                                        QUEUE_PATH_MAPPING.remove(key);
1425                                }
1426                        }
1427                }
1428
1429                if (QUEUE_PATH_MAPPING.keySet().size() >= RemoteFile.MAX_QUEUE_SIZE)
1430                        return false;
1431
1432                return true;
1433        }
1434
1435        protected static synchronized void removeQueuePathMapping(String path) {
1436                QUEUE_PATH_MAPPING.remove(path);
1437        }
1438
1439        public static synchronized Map<String, Long> getQueuePathMapping() {
1440                return Collections.synchronizedMap(QUEUE_PATH_MAPPING);
1441        }
1442
1443        private static void debugLog(String point,RemoteFile rf,CacheArrayFilter filter){
1444                debugLog(DiskFile.class,point,rf,filter);
1445        }
1446        
1447}