001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import java.nio.file.Paths;
028import java.nio.file.Files;
029import java.io.IOException;
030import java.nio.file.OpenOption;
031import java.nio.file.StandardOpenOption;
032import java.nio.channels.FileChannel;
033import java.nio.channels.FileLock;
034import java.nio.file.Path;
035import java.nio.ByteBuffer;
036import com.killcoding.log.Logger;
037import java.net.URI;
038import java.util.Comparator;
039import java.util.List;
040import java.util.ArrayList;
041import java.nio.channels.SeekableByteChannel;
042import java.io.ByteArrayOutputStream;
043import java.nio.file.attribute.BasicFileAttributes;
044import java.nio.file.attribute.FileTime;
045import java.time.Instant;
046import java.util.Arrays;
047import java.io.ByteArrayInputStream;
048import java.sql.Timestamp;
049import java.util.stream.Stream;
050import java.util.stream.Collectors;
051import java.util.HashSet;
052import java.util.Set;
053import java.nio.file.DirectoryStream;
054import java.nio.file.CopyOption;
055import java.nio.file.LinkOption;
056import com.killcoding.tool.CommonTools;
057import com.killcoding.file.RemoteFile;
058import com.killcoding.log.LoggerFactory;
059import java.util.concurrent.Future;
060import com.killcoding.tool.ConfigProperties;
061import com.killcoding.datasource.Clock;
062import java.util.concurrent.Callable;
063import com.killcoding.file.DiskFile;
064import java.nio.file.attribute.FileAttribute;
065import java.text.SimpleDateFormat;
066import com.killcoding.cache.CacheArray;
067import com.killcoding.cache.CacheArrayFilter;
068import java.util.Map;
069import java.util.concurrent.ExecutorService;
070import java.util.concurrent.Executors;
071import java.nio.file.NoSuchFileException;
072import java.util.concurrent.ConcurrentHashMap;
073import java.util.Iterator;
074import java.util.Date;
075import java.time.ZonedDateTime;
076import java.time.LocalDateTime;
077import java.time.ZoneId;
078import java.time.format.DateTimeFormatter;
079import java.util.Calendar;
080import java.util.TimeZone;
081import java.text.ParseException;
082import java.util.Collections;
083import java.util.HashMap;
084import java.nio.file.FileAlreadyExistsException;
085import java.util.LinkedList;
086
087public class DiskFile extends BaseFile {
088    
089    public static final int RETRY_OPEN_LIMITED = 100;
090
091        protected static Integer MAX_POOL_SIZE = 100;
092        private static final Map<String, Long> QUEUE_PATH_MAPPING = new ConcurrentHashMap<String, Long>();
093
094        private static ExecutorService splitPool = null;
095
096        protected boolean copyStructureOnly = false;
097        private FileChannel channel = null;
098        private FileLock lock = null;
099        private long modifyTimeMs = 0L;
100
101        public DiskFile(String path) {
102                super(path);
103        }
104
105        public static synchronized void initPool(int poolSize) {
106                if (splitPool == null) {
107                        MAX_POOL_SIZE = poolSize;
108                        LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
109                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
110                }
111        }
112
113        private static synchronized void initDefaultPool() {
114                if (splitPool == null) {
115                        LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
116                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
117                }
118        }
119
120        public void split(int partSize, FilePart filePart) throws IOException {
121                if (isLink()) {
122                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
123                }
124                if (isDir()) {
125                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
126                }
127                if (!exists()) {
128                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
129                }
130                initDefaultPool();
131                splitPool.execute(new Runnable() {
132                        @Override
133                        public void run() {
134                                SeekableByteChannel ch = null;
135                                int partIndex = -1;
136                                long fileSize = 0L;
137                                try {
138                                        Path originPath = Paths.get(origin.toURI());
139                                        Thread.currentThread().setName(String.format("DiskFile-split-%s", origin.getName()));
140                                        ch = Files.newByteChannel(originPath, StandardOpenOption.READ);
141                                        ByteBuffer bf = ByteBuffer.allocate(partSize);
142                                        while (ch.read(bf) != -1) {
143                                                bf.flip();
144                                                partIndex += 1;
145                                                List<Byte> parts = new ArrayList<Byte>();
146                                                while (bf.hasRemaining()) {
147                                                        parts.add(bf.get());
148                                                }
149                                                int partsSize = parts.size();
150                                                fileSize += partsSize;
151                                                byte[] part = new byte[partsSize];
152                                                for (int i = 0; i < partsSize; i++) {
153                                                        part[i] = parts.get(i);
154                                                }
155                                                filePart.process(partIndex, part);
156                                                bf.clear();
157                                        }
158                                        filePart.completed(partIndex, getModifyTimeForClock(), fileSize);
159                                } catch (Exception e) {
160                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
161                                } finally {
162                                        if (ch != null) {
163                                                try {
164                                                        ch.close();
165                                                } catch (IOException e) {
166                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
167                                                }
168                                        }
169                                        filePart.ended(partIndex, fileSize);
170                                }
171                        }
172                });
173        }
174
175        @Override
176        public void copyTo(String toPath) throws IOException {
177                Path originPath = Paths.get(origin.toURI());
178                Path destPath = Paths.get(toPath);
179                LoggerFactory.getLogger(DiskFile.class).debug("CopyTo: {} -> {}", originPath, destPath);
180                Files.copy(originPath, destPath);
181        }
182
183        @Override
184        public void moveTo(String toPath) throws IOException {
185                Path originPath = Paths.get(origin.toURI());
186                Path destPath = Paths.get(toPath);
187                LoggerFactory.getLogger(DiskFile.class).debug("MoveTo: {} -> {}", originPath, destPath);
188                if (Files.exists(destPath)) {
189                        Files.delete(destPath);
190                }
191                Files.move(originPath, destPath);
192        }
193
194        public RemoteFile writeToRemote(RemoteFile remoteFile) throws IOException {
195                return writeToRemote(remoteFile, new Runnable() {
196                        @Override
197                        public void run() {
198
199                        }
200                });
201        }
202
203        public RemoteFile writeToRemote(RemoteFile remoteFile, Runnable completedCallback) throws IOException {
204                if (remoteFile.exists(true)) {
205                        throw new IOException(String.format("The remote file '%s' already exists.", remoteFile.getPath()));
206                } else {
207                        remoteFile.copyFrom(copyStructureOnly, this);
208                        if (isDir()) {
209                                startFullAsync(0, this.getPath() + "/", remoteFile.getPath() + "/", completedCallback, null);
210                        }
211                }
212                return remoteFile;
213        }
214
215        protected boolean isLogicModify() throws Exception {
216                boolean b = false;
217                String tmpPath = toModifyTmpFilePath();
218
219                DiskFile df = null;
220                Path logicPath = Paths.get(tmpPath);
221                if (Files.exists(logicPath)) {
222                        df = new DiskFile(tmpPath);
223                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
224                        Timestamp dfModifyTime = df.getModifyTimeForClock();
225
226                        long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
227
228                        b = diffMs < LOGIC_TIMEOUT_MS;
229                }
230                return b;
231        }
232
233        protected DiskFile logicModify() throws Exception {
234                return logicModify(null);
235        }
236
237        protected DiskFile logicModify(String msg) throws Exception {
238                if (getParentFile().exists()) {
239                        String tmpLogicPath = toModifyTmpFilePath();
240
241                        if (tmpLogicPath == null)
242                                return null;
243
244                        Path logicPath = Paths.get(tmpLogicPath);
245                        DiskFile df = new DiskFile(tmpLogicPath);
246                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
247
248                        if (msg == null) {
249                                df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
250                                                false);
251                        } else {
252                                df.write(msg, false);
253                        }
254
255                        Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
256                        return df;
257                }
258                return null;
259        }
260
261        protected void removeLogicModify() {
262                try {
263                        String tmpPath = toModifyTmpFilePath();
264                        if (tmpPath != null) {
265                                DiskFile df = new DiskFile(tmpPath);
266                                DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
267                                if (df.exists()) {
268                                        df.delete();
269                                }
270                        }
271                } catch (Exception e) {
272                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
273                }
274        }
275
276        protected boolean isLogicAccess() throws Exception {
277            boolean b = false;
278                String tmpPath = toAccessTmpFilePath();
279                DiskFile df = null;
280                Path logicPath = Paths.get(tmpPath);
281                if (Files.exists(logicPath)) {
282                        df = new DiskFile(tmpPath);
283                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
284                        Timestamp dfModifyTime = df.getModifyTimeForClock();
285
286                        long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
287
288                        b = diffMs < LOGIC_ACCESS_TIMEOUT_MS;
289                }
290                return b;
291        }
292
293        protected DiskFile logicAccess() throws Exception {
294                return logicAccess(null);
295        }
296
297        protected DiskFile logicAccess(String msg) throws Exception {
298                if (getParentFile().exists()) {
299                        String tmpLogicPath = toAccessTmpFilePath();
300                        Path logicPath = Paths.get(tmpLogicPath);
301                        DiskFile df = new DiskFile(tmpLogicPath);
302                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
303                        if (msg == null) {
304                                df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
305                                                false);
306                        } else {
307                                df.write(msg, false);
308                        }
309
310                        Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
311                        return df;
312                }
313                return null;
314        }
315
316        protected void removeLogicAccess() {
317                try {
318                        String tmpPath = toAccessTmpFilePath();
319                        if (tmpPath != null) {
320                                DiskFile df = new DiskFile(tmpPath);
321                                DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
322                                if (df.exists()) {
323                                        df.delete();
324                                }
325                        }
326                } catch (Exception e) {
327                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
328                }
329        }
330
331        protected boolean isLogicCheck() throws Exception {
332                boolean b = false;
333                String tmpPath = toCheckTmpFilePath();
334                
335                DiskFile df = null;
336                Path logicPath = Paths.get(tmpPath);
337                if (Files.exists(logicPath)) {
338                        df = new DiskFile(tmpPath);
339                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
340                        Timestamp dfModifyTime = df.getModifyTimeForClock();
341                                
342                        long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
343
344                        b = diffMs < LOGIC_CHECK_TIMEOUT_MS;
345                }
346                return b;
347        }
348
349        protected DiskFile logicCheck() throws Exception {
350                return logicCheck(null);
351        }
352
353        public DiskFile logicCheck(String msg) throws Exception {
354        if (getParentFile().exists()) {
355                String tmpLogicPath = toCheckTmpFilePath();
356                if (tmpLogicPath == null)
357                        return null;
358    
359                Path logicPath = Paths.get(tmpLogicPath);
360                DiskFile df = new DiskFile(tmpLogicPath);
361                DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
362                if (msg == null) {
363                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
364                                        false);
365                } else {
366                        df.write(msg, false);
367                }
368    
369                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
370                return df;
371        }
372                return null;
373        }
374
375        protected void removeLogicCheck() {
376            try{
377                String tmpPath = toCheckTmpFilePath();
378                if (tmpPath != null) {
379                        DiskFile df = new DiskFile(tmpPath);
380                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
381                        if (df.exists()) {
382                                df.delete();
383                                DiskFile pdf = new DiskFile(this.getOrigin().getParent());
384                                DiskFile.copyAttrs(pdf, this.copyStructureOnly, this.syncRoot);
385                                if(pdf.isLogicCheck()){
386                                    pdf.removeLogicCheck();
387                                }
388                        }
389                }
390            }catch(Exception e){
391                LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
392            }
393        }
394
395        @Override
396        public boolean beforeDelete(boolean realDeleted) {
397                return true;
398        }
399
400        @Override
401        public void afterDelete(boolean realDeleted) {
402
403        }
404
405        @Override
406        public boolean beforeMkdirs() {
407                return true;
408        }
409
410        @Override
411        public void afterMkdirs() {
412
413        }
414
415        @Override
416        public boolean beforeCreateLink(String target) {
417                return true;
418        }
419
420        @Override
421        public void afterCreateLink(String target) {
422
423        }
424
425        @Override
426        public boolean beforeWrite() {
427                return true;
428        }
429
430        @Override
431        public void afterWrite() {
432
433        }
434
435        @Override
436        public boolean delete() throws IOException {
437                return deleteFrom();
438        }
439
440        private boolean deleteFrom() throws IOException {
441                boolean allowed = beforeDelete(true);
442                if (allowed) {
443                        LoggerFactory.getLogger(DiskFile.class).debug("Delete: {}", getPath());
444                        if (isFile()) {
445                                Path originPath = Paths.get(origin.toURI());
446                                boolean exists = Files.exists(originPath);
447                                if (exists) {
448                                        Files.deleteIfExists(Paths.get(origin.toURI()));
449                                        afterDelete(true);
450                                        return true;
451                                }
452                        }
453                        if (isDir()) {
454                                Path originPath = Paths.get(origin.toURI());
455                                boolean exists = Files.exists(originPath);
456                                if (exists) {
457                                        Files.walk(originPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
458                                        afterDelete(true);
459                                        return true;
460                                }
461                        }
462                        if (isLink()) {
463                                Path originPath = Paths.get(origin.toURI());
464                                boolean exists = Files.exists(originPath);
465                                if (exists) {
466                                        Files.deleteIfExists(Paths.get(origin.toURI()));
467                                        afterDelete(true);
468                                        return true;
469                                }
470                        }
471                }
472                return false;
473        }
474
475        @Override
476        public boolean isFile() throws IOException {
477                return this.origin.isFile() && !isLink();
478        }
479
480        @Override
481        public boolean isDirectory() throws IOException {
482                return this.origin.isDirectory() && !isLink();
483        }
484
485        @Override
486        public boolean isLink() throws IOException {
487                return Files.isSymbolicLink(Paths.get(this.origin.toURI()));
488        }
489
490        @Override
491        public boolean exists() throws IOException {
492                return this.origin.exists();
493        }
494
495        @Override
496        public boolean mkdirs() throws IOException {
497                boolean allowed = beforeMkdirs();
498                if (allowed) {
499                        LoggerFactory.getLogger(DiskFile.class).debug("Mkidrs: {}", getPath());
500                        boolean b = this.origin.mkdirs();
501                        if (b && modifyTimeMs > 0) {
502                                setModifyTime(new Timestamp(modifyTimeMs));
503                        }
504                        afterMkdirs();
505                        return b;
506                }
507                return false;
508        }
509
510        @Override
511        public boolean write(byte[] data, boolean append) throws IOException {
512
513                boolean allowed = beforeWrite();
514
515                if (!allowed)
516                        return false;
517
518                if (data == null)
519                        data = new byte[] {};
520
521                try {
522
523                        OpenOption[] options = null;
524                        if (append) {
525                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
526                                                StandardOpenOption.APPEND };
527                        } else {
528                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
529                                                StandardOpenOption.TRUNCATE_EXISTING };
530                        }
531
532                        File parent = origin.getParentFile();
533                        if (!parent.exists()) {
534                                parent.mkdirs();
535                        }
536                        int retryCount = 0;
537                        Path path = Paths.get(origin.toURI());
538                        while(retryCount < RETRY_OPEN_LIMITED){
539                            retryCount++;
540                            try{
541                            channel = FileChannel.open(path, options);
542                                lock = channel.lock();
543                                if (lock != null) {
544                                        ByteBuffer buf = ByteBuffer.wrap(data);
545                                        channel.write(buf);
546                                        if (modifyTimeMs > 0) {
547                                                setModifyTime(new Timestamp(modifyTimeMs));
548                                        }
549                                        return true;
550                                }
551                            }catch(Exception e){
552                                LoggerFactory.getLogger(DiskFile.class).debug(e.getMessage(),e);
553                                
554                                if(retryCount >= RETRY_OPEN_LIMITED) throw new IOException(e.getMessage(),e);
555                                
556                            }
557                        }
558                        return false;
559                } catch (Exception e) {
560                        throw e;
561                } finally {
562                        close();
563                }
564        }
565
566        public boolean manualOpen(boolean append) throws IOException, FileAlreadyExistsException {
567
568                boolean allowed = beforeWrite();
569
570                if (!allowed)
571                        return false;
572
573                OpenOption[] options = null;
574                if (append) {
575                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
576                                        StandardOpenOption.APPEND };
577                } else {
578                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
579                                        StandardOpenOption.TRUNCATE_EXISTING };
580                }
581                Path path = Paths.get(origin.toURI());
582                File parent = origin.getParentFile();
583                if (parent != null && !parent.exists() && !parent.mkdirs()) {
584                        throw new IOException(String.format("Failed to create parent directory: %s", parent));
585                }
586                if (!Files.exists(path)) {
587                        try {
588                                Files.createFile(path);
589                        } catch (FileAlreadyExistsException e) {
590                                throw e;
591                        }
592                }
593                int retryCount = 0;
594                while(retryCount < RETRY_OPEN_LIMITED){
595                    retryCount++;
596                    try{
597                        channel = FileChannel.open(path, options);
598                        lock = channel.lock();
599                        return lock != null;
600                    }catch(Exception e){
601                        LoggerFactory.getLogger(DiskFile.class).debug(e.getMessage(),e);
602                        
603                        if(retryCount >= RETRY_OPEN_LIMITED) throw new IOException(e.getMessage(),e);
604                    }
605                }
606                return false;
607        }
608
609        public void manualWrite(byte[] data) throws IOException {
610                if (data == null)
611                        data = new byte[] {};
612
613                if (lock != null) {
614                        ByteBuffer buf = ByteBuffer.wrap(data);
615                        channel.write(buf);
616                        if (modifyTimeMs > 0) {
617                                setModifyTime(new Timestamp(modifyTimeMs));
618                        }
619                }
620        }
621
622        public void manualWrite(String data) throws IOException {
623                manualWrite(data.getBytes(CHARSET));
624        }
625
626        public void manualClose() throws IOException {
627                close();
628        }
629
630        @Override
631        public boolean write(byte[] data) throws IOException {
632                return write(data, false);
633        }
634
635        @Override
636        public boolean write(String data, boolean append) throws IOException {
637                return write(data.getBytes(CHARSET), append);
638        }
639
640        public boolean write(String data, String charset, boolean append) throws IOException {
641                return write(data.getBytes(charset), append);
642        }
643
644        @Override
645        public boolean write(String data) throws IOException {
646                return write(data.getBytes(CHARSET));
647        }
648
649        public boolean write(String data, String charset) throws IOException {
650                return write(data.getBytes(charset));
651        }
652
653        @Override
654        public boolean createLink(String target) throws IOException {
655                boolean allowed = beforeCreateLink(target);
656                if (allowed) {
657                        LoggerFactory.getLogger(DiskFile.class).debug("CreateLink: {}", getPath());
658                        try {
659                                Path link = Paths.get(origin.toURI());
660                                Path targetPath = Paths.get(target);
661                                Files.createSymbolicLink(link, targetPath);
662                                if (modifyTimeMs > 0) {
663                                        setModifyTime(new Timestamp(modifyTimeMs));
664                                }
665                                afterCreateLink(target);
666                                return true;
667                        } catch (IOException e) {
668                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
669                        }
670                }
671                return false;
672        }
673
674        @Override
675        public String readLink() throws IOException {
676                Path link = Paths.get(origin.toURI());
677                return Files.readSymbolicLink(link).toString();
678        }
679
680        @Override
681        public byte[] readAllBytes() throws IOException {
682                return readAllBytesFrom();
683        }
684
685        private byte[] readAllBytesFrom() throws IOException {
686                if (isLink()) {
687                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
688                }
689                if (isDir()) {
690                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
691                }
692                if (!exists()) {
693                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
694                }
695                return Files.readAllBytes(Paths.get(getPath()));
696        }
697
698        @Override
699        public String readAllString() throws IOException {
700                return readAllString(BaseFile.CHARSET);
701        }
702
703        public String readAllString(String charset) throws IOException {
704                ByteArrayOutputStream baos = null;
705                try {
706                        byte[] bytes = readAllBytes();
707                        baos = new ByteArrayOutputStream();
708                        baos.write(bytes);
709                        baos.flush();
710                        return baos.toString(charset);
711                } finally {
712                        if (baos != null) {
713                                try {
714                                        baos.close();
715                                } catch (IOException e) {
716                                        Logger.systemError(DiskFile.class, e.getMessage(), e);
717                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
718                                }
719                        }
720                }
721        }
722
723        @Override
724        public long size() throws IOException {
725                if (isFile()) {
726                        Path path = Paths.get(origin.getAbsolutePath());
727                        return Files.size(path);
728                }
729                return 0L;
730        }
731
732        private void close() {
733                if (lock != null) {
734                        try {
735                                lock.release();
736                        } catch (IOException e) {
737                                Logger.systemError(DiskFile.class, e.getMessage(), e);
738                        }
739                        try {
740                                lock.close();
741                        } catch (IOException e) {
742                                Logger.systemError(DiskFile.class, e.getMessage(), e);
743                        } finally {
744                                lock = null;
745                        }
746                }
747
748                if (channel != null) {
749                        try {
750                                channel.close();
751                        } catch (IOException e) {
752                                Logger.systemError(DiskFile.class, e.getMessage(), e);
753                        } finally {
754                                channel = null;
755                        }
756                }
757                afterWrite();
758        }
759
760        @Override
761        public boolean complete() throws IOException {
762                close();
763                return true;
764        }
765
766        public void setModifyTimeMs(long modifyTimeMs) throws IOException {
767                this.modifyTimeMs = modifyTimeMs;
768        }
769
770        public void setModifyTimeMsFromClock(long modifyTimeMsFromClock) throws IOException {
771                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
772                String localTzId = Calendar.getInstance().getTimeZone().getID();
773                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, new Timestamp(modifyTimeMsFromClock));
774                this.modifyTimeMs = fileLocalTimestamp.getTime();
775        }
776
777        @Override
778        public void setModifyTime(Timestamp modifyTime) throws IOException {
779                if (exists()) {
780                        Path originPath = Paths.get(origin.toURI());
781                        Files.setLastModifiedTime(originPath, FileTime.fromMillis(modifyTime.getTime()));
782                }
783        }
784
785        public void setModifyTimeFromClock(Timestamp modifyTimeFromClock) throws IOException {
786                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
787                String localTzId = Calendar.getInstance().getTimeZone().getID();
788                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, modifyTimeFromClock);
789                setModifyTime(fileLocalTimestamp);
790        }
791
792        @Override
793        public Timestamp getModifyTime() throws IOException {
794                Path originPath = Paths.get(origin.toURI());
795                if (Files.exists(originPath)) {
796                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
797                        FileTime fileTime = attr.lastModifiedTime();
798                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
799                        return fileLocalTimestamp;
800                }
801                return null;
802        }
803
804        public Timestamp getModifyTimeForClock() throws IOException {
805                Path originPath = Paths.get(origin.toURI());
806                if (Files.exists(originPath)) {
807                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
808                        FileTime fileTime = attr.lastModifiedTime();
809                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
810                        String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
811                        String localTzId = Calendar.getInstance().getTimeZone().getID();
812                        Timestamp fileRemoteTimestamp = timeZoneConver(remoteTzId, localTzId, fileLocalTimestamp);
813                        return fileRemoteTimestamp;
814                }
815                return null;
816        }
817
818        public void startFullAsync(long timerMs, String replaceTo) throws IOException {
819                startFullAsync(timerMs, getPath() + "/", replaceTo, null, null);
820        }
821
822        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback) throws IOException {
823                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, null);
824        }
825
826        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback,
827                        Runnable checkDeleteCompletedCallback) throws IOException {
828                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, checkDeleteCompletedCallback);
829        }
830
831        private void startFullAsync(long timerMs, String replaceRootPath, String replaceTo, Runnable completedCallback,
832                        Runnable checkDeleteCompletedCallback) throws IOException {
833
834                if (!RemoteFile.FIRST_LOADED) {
835                        RemoteFile firstLoaded = new RemoteFile(replaceTo);
836                }
837
838                Runnable runnable = new Runnable() {
839                        @Override
840                        public void run() {
841                                try {
842                                        startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
843                                } catch (Exception e) {
844                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
845                                }
846                        }
847                };
848                Executors.newFixedThreadPool(1).execute(runnable);
849
850                runnable = new Runnable() {
851                        @Override
852                        public void run() {
853                                try {
854                                        startFullSyncForDelete(-1, timerMs, replaceTo, replaceRootPath, checkDeleteCompletedCallback);
855                                } catch (Exception e) {
856                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
857                                }
858                        }
859                };
860                Executors.newFixedThreadPool(1).execute(runnable);
861        }
862
863        private synchronized void startFullSyncForDelete(final Integer filePartDataTable, final long timerMs,
864                        final String replaceTo, final String replaceRootPath, final Runnable completedCallback) throws IOException {
865                DiskFile the = this;
866                RemoteFile rootDir = null;
867                if (!CommonTools.isBlank(replaceTo)) {
868                        rootDir = new RemoteFile(replaceTo);
869                }
870                if (rootDir == null) {
871                        throw new IOException("The root folder is null.");
872                }
873                RemoteFile _rootDir = rootDir;
874                Integer maxDataTable = rootDir.getMaxFilePartDataTable();
875
876                Integer _filePartDataTable = filePartDataTable;
877
878                if (filePartDataTable > maxDataTable)
879                        _filePartDataTable = -1;
880
881                final Integer finalFilePartDataTable = _filePartDataTable;
882
883                final CacheArray rows = new CacheArray();
884
885                rows.filter(getScanDeleteFilter(rootDir, finalFilePartDataTable, timerMs, replaceTo, replaceRootPath,
886                                completedCallback));
887
888                Runnable runnable = new Runnable() {
889                        @Override
890                        public void run() {
891                                try {
892                                        _rootDir.listAllForScanDelete(finalFilePartDataTable, rows);
893                                } catch (Exception e) {
894                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
895                                }
896                        }
897                };
898                Executors.newFixedThreadPool(1).execute(runnable);
899        }
900
901        private CacheArrayFilter getScanDeleteFilter(final RemoteFile rootDir, final Integer filePartDataTable,
902                        final long timerMs, final String replaceTo, final String replaceRootPath,
903                        final Runnable completedCallback) {
904                DiskFile the = this;
905                CacheArrayFilter filter = new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
906                        @Override
907                        public void execute(Integer index, Object o) {
908
909                                try {
910                                        Map<String, Object> item = (Map<String, Object>) o;
911                                        Object id = item.get("id");
912                                        String parent = (String) item.get("file_parent_path");
913                                        String fileName = (String) item.get("file_name");
914                                        String pf = String.format("%s/%s", parent, fileName);
915                                        RemoteFile rf = new RemoteFile(pf);
916                                        Thread.currentThread()
917                                                        .setName(String.format("DiskFile-startFullSyncForDelete-(DataTable=%s Index=%s)-%s",
918                                                                        filePartDataTable, index, fileName));
919
920                                        if (memoryUsage() < RemoteFile.MAX_MEMORY_USAGE) {
921                                                String toDf = pf;
922                                                if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
923                                                        toDf = pf.replaceFirst(replaceTo + "[/]*", replaceRootPath + "/");
924                                                }
925
926                                                DiskFile df = new DiskFile(toDf);
927                                                DiskFile.copyAttrs(df, the.copyStructureOnly, the.syncRoot);
928                                                boolean isDebug = isDebug(rf);
929                                                boolean isRoot = rootDir.getPath().equals(rf.getPath());
930                                                boolean isSyncedHost = rf.isSyncedOnHostname(id);
931                                                boolean rfExists = rf.exists();
932
933                                                boolean allowedForceDelete = rootDir.exists() && !isRoot && isSyncedHost && rfExists
934                                                                && !df.exists() && !df.isLogicModify();
935
936                                                if (allowedForceDelete) {
937                                                        df.logicModify();
938                                                        if (isDebug) {
939                                                                LoggerFactory.getLogger(DiskFile.class).mark("ForceRemoteDelete - {}", rf.getPath());
940                                                        }
941                                                        rf.forceDeleteAll(true);
942                                                        df.removeLogicModify();
943                                                }
944                                        } else {
945                                                LoggerFactory.getLogger(DiskFile.class).warn("Over max memory usage 'MAX_MEMORY_USAGE={}'.",
946                                                                RemoteFile.MAX_MEMORY_USAGE);
947                                        }
948                                } catch (Exception e) {
949                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
950                                }
951                        }
952
953                        @Override
954                        public void completed(Integer size) {
955                                if (isStopSync()) {
956                                        LoggerFactory.getLogger(DiskFile.class).mark("Stoped start full sync for delete.");
957                                }
958                                if (timerMs <= 0) {
959                                        LoggerFactory.getLogger(DiskFile.class).mark("Completed start full sync for delete.");
960                                }
961
962                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
963
964                                if (isRootCompleted && completedCallback != null) {
965                                        completedCallback.run();
966                                }
967
968                                if (!isStopSync() && timerMs > 0) {
969                                        while (true) {
970
971                                                if (isStopSync())
972                                                        break;
973
974                                                try {
975                                                        Thread.sleep(timerMs);
976                                                } catch (InterruptedException e) {
977                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
978                                                }
979                                                boolean allowed = checkUsage(rootDir);
980                                                if (allowed) {
981                                                        try {
982                                                                if (!isStopSync()) {
983                                                                        startFullSyncForDelete(filePartDataTable + 1, timerMs, replaceTo, replaceRootPath,
984                                                                                        completedCallback);
985                                                                }
986                                                        } catch (IOException e) {
987                                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
988                                                        }
989                                                        break;
990                                                }
991                                        }
992                                }
993                        }
994                };
995                return filter;
996        }
997
998        private void startFullAsyncThread(final long timerMs, String replaceRootPath, String replaceTo,
999                        final Runnable completedCallback) {
1000                stopSync = false;
1001                this.setCopyStructureOnly(RemoteFile.COPY_STRUCTURE_ONLY);
1002                final DiskFile the = this;
1003                Thread.currentThread().setName(String.format("DiskFile-startFullAsyncThread-%s", the.getOrigin().getName()));
1004                replaceRootPath = replacePath(replaceRootPath);
1005                replaceTo = replacePath(replaceTo);
1006                DiskFile rootDisk = new DiskFile(replaceRootPath);
1007                boolean rootExists = false;
1008                try {
1009                        rootExists = rootDisk.exists();
1010                } catch (Exception e) {
1011                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1012                }
1013                if (!rootExists) {
1014                        Logger.systemError(DiskFile.class, "The root path '{}' does not exist.", rootDisk.getPath());
1015                }
1016                boolean isRoot = the.getPath().equals(rootDisk.getPath());
1017                LoggerFactory.getLogger(DiskFile.class).debug("IsRoot: {} - {}", isRoot, the.getPath());
1018                LoggerFactory.getLogger(DiskFile.class).debug("CacheArrayUsage: {}", CacheArray.getUsage());
1019                if (rootExists) {
1020
1021                        DirectoryStream<Path> stream = null;
1022                        try {
1023                                if (syncRoot == null) {
1024                                        File replaceRootFile = new File(replaceRootPath);
1025                                        if (replaceRootFile.getParent() == null) {
1026                                                throw new IOException("Cannot sync the root path.");
1027                                        }
1028                                        String replaceRootParent = replacePath(replaceRootFile.getParent());
1029                                        syncRoot = new File(String.format("%s/%s", replaceRootParent, replaceRootFile.getName()));
1030                                }
1031
1032                                if (!CommonTools.isBlank(replaceTo)) {
1033                                        RemoteFile rootDir = new RemoteFile(replaceTo);
1034                                        if (!rootDir.exists()) {
1035                                                rootDir.setModifyTime(this.getModifyTimeForClock());
1036                                                rootDir.mkdirs();
1037                                        }
1038                                }
1039                                Path originPath = Paths.get(origin.toURI());
1040                                CacheArray rows = new CacheArray();
1041                                CacheArrayFilter filter = getFullAsyncFilter(the, timerMs, replaceRootPath, replaceTo,
1042                                                completedCallback);
1043                                rows.filter(filter);
1044                                stream = Files.newDirectoryStream(originPath);
1045                                for (Path p : stream) {
1046                                        if (p != null) {
1047                                                File f = p.toFile();
1048                                                boolean isFile = f.isFile();
1049                                                if (isFile) {
1050                                                        boolean isWritingDf = f.getName().matches(TMP_WRITING_SWP);
1051                                                        if (isWritingDf) {
1052                                                                BasicFileAttributes attributes = Files.readAttributes(p, BasicFileAttributes.class);
1053                                                                FileTime creationTime = attributes.creationTime();
1054                                                                boolean timeouted = (Calendar.getInstance().getTimeInMillis()
1055                                                                                - creationTime.toMillis()) > (RemoteFile.BATCH_SIZE * LOGIC_TIMEOUT_MS);
1056                                                                if (timeouted) {
1057                                                                        try {
1058                                                                                boolean exists = Files.exists(p);
1059                                                                                if (exists) {
1060                                                                                        Files.deleteIfExists(p);
1061                                                                                }
1062                                                                        } catch (Exception e) {
1063                                                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1064                                                                        }
1065                                                                }
1066                                                        }
1067                                                }
1068                                                boolean allowedPath = the.checkSyncPathAvailable(p.toString());
1069                                                if (!allowedPath) {
1070                                                        LoggerFactory.getLogger(DiskFile.class).debug("Ignored: {}", p.toString());
1071                                                }
1072                                                if (allowedPath) {
1073                                                        rows.add(p);
1074                                                }
1075                                        }
1076                                }
1077                                rows.add(null);
1078                        } catch (Exception e) {
1079                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1080                                removeLogicAccess();
1081                                if (!(e instanceof NoSuchFileException)) {
1082                                        if (!isStopSync() && timerMs > 0) {
1083                                                try {
1084                                                        Thread.sleep(timerMs);
1085                                                } catch (InterruptedException ee) {
1086                                                        LoggerFactory.getLogger(DiskFile.class).error(ee.getMessage(), ee);
1087                                                }
1088                                                if (!isStopSync()) {
1089                                                        startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
1090                                                }
1091                                        }
1092                                }
1093                        } finally {
1094                                if (stream != null) {
1095                                        try {
1096                                                stream.close();
1097                                        } catch (IOException e) {
1098                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1099                                        }
1100                                }
1101                        }
1102                }
1103        }
1104
1105        private CacheArrayFilter getFullAsyncFilter(DiskFile the, long timerMs, String replaceRootPath, String replaceTo,
1106                        Runnable completedCallback) {
1107
1108                return new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
1109                        RemoteFile rf = null;
1110                        final CacheArrayFilter theFilter = this;
1111
1112                        @Override
1113                        public void completed(Integer size) {
1114                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1115
1116                                if (isRootCompleted) {
1117                                        try {
1118                                                if (rf != null && the.isStopSync() && isDebug(rf)) {
1119                                                        LoggerFactory.getLogger(DiskFile.class).mark("StopedRootScan - {}", the.getPath());
1120                                                }
1121                                                if (!the.isStopSync() && timerMs > 0) {
1122                                                        Thread.sleep(timerMs);
1123                                                        if (!the.isStopSync()) {
1124                                                                if (rf != null && isDebug(rf)) {
1125                                                                        LoggerFactory.getLogger(DiskFile.class).mark("RootScaning - {}", the.getPath());
1126                                                                }
1127                                                                the.startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
1128                                                        }
1129                                                }
1130                                                if (completedCallback != null) {
1131                                                        completedCallback.run();
1132                                                }
1133                                        } catch (Exception e) {
1134                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1135                                        }
1136                                }
1137                        }
1138
1139                        @Override
1140                        public void execute(Integer index, Object o) {
1141                                DiskFile df = null;
1142                                try {
1143                                        Path p = (Path) o;
1144                                        String pf = replacePath(p.toFile().getAbsolutePath());
1145                                        Thread.currentThread().setName(
1146                                                        String.format("DiskFile-getFullAsyncFilter-%s", p.toFile().getParentFile().getName()));
1147                                        String toRf = pf;
1148                                        if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
1149                                                toRf = pf.replaceFirst(replaceRootPath + "[/]*", replaceTo + "/");
1150                                        }
1151
1152                                        df = new DiskFile(pf);
1153                                        DiskFile.copyAttrs(df, the.copyStructureOnly, the.syncRoot);
1154                                        rf = new RemoteFile(toRf);
1155
1156                                        boolean isRoot = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1157
1158                                        boolean allowedConn = checkUsage(rf);
1159                                        String fileName = p.toFile().getName();
1160                                        boolean isDebug = isDebug(rf);
1161
1162                                        boolean timeouted = rf.isTimeout();
1163                                        if (timeouted && RemoteFile.addQueuePathMapping(rf.getPath()) && allowedConn) {
1164                                                if (isDebug) {
1165                                                        LoggerFactory.getLogger(DiskFile.class).mark("ForceDelete(Timeouted) - {}", rf.getPath());
1166                                                }
1167                                                rf.forceDeleteFile(false);
1168                                                RemoteFile.removeQueuePathMapping(rf.getPath());
1169                                        }
1170                                        Timestamp modifyTime = df.getModifyTimeForClock();
1171                                        boolean allowedModifyTime = modifyTime != null;
1172                                        boolean allowed = DiskFile.checkQueuePathMapping() && !isStopSync() && allowedModifyTime
1173                                                        && allowedConn;
1174
1175                                        if (isDebug) {
1176                                                debugLog("CheckUsage", rf, this);
1177                                        }
1178
1179                                        if (!isRoot && allowed) {
1180                                                allowed = !df.isLogicCheck();
1181                                        }
1182
1183                                        if (allowed) {
1184
1185                                                if (!isRoot)
1186                                                        df.logicCheck();
1187
1188                                                rf.setModifyTime(modifyTime);
1189                                                long dfModifyTimeMs = modifyTime.getTime();
1190                                                boolean blocked = false;
1191                                                boolean isRealDelete = rf.isLastOperateRealDelete();
1192
1193                                                if (isRealDelete) {
1194                                                        String clientHostname = CommonTools.getHostname();
1195                                                        String delHostname = rf.getLastSourceHostname();
1196                                                        boolean isOwnerDel = clientHostname.equals(delHostname);
1197                                                        if (isOwnerDel) {
1198                                                                Timestamp deletedTime = rf.getLastOperateTime();
1199                                                                if (deletedTime != null) {
1200                                                                        boolean waiting = new Clock().getTime() - deletedTime.getTime() <= LOGIC_TIMEOUT_MS;
1201                                                                        if (waiting) {
1202                                                                                blocked = true;
1203                                                                        }
1204                                                                }
1205                                                        } else {
1206                                                                Timestamp deletedTime = rf.getLastOperateTime();
1207                                                                if (deletedTime != null) {
1208                                                                        boolean olded = modifyTime.getTime() < deletedTime.getTime();
1209                                                                        if (olded) {
1210                                                                                blocked = true;
1211                                                                        }
1212                                                                }
1213                                                        }
1214                                                }
1215
1216                                                boolean haveParent = rf.getParentFile().exists();
1217                                                if (!blocked && haveParent) {
1218                                                        if (df.isLink()) {
1219                                                                if (!df.isLogicAccess()) {
1220                                                                        df.logicAccess();
1221                                                                        if (rf.exists()) {
1222                                                                                if (rf.isLink()) {
1223                                                                                        long rfModifyTimeMs = rf.getModifyTime().getTime();
1224                                                                                        boolean changedTime = dfModifyTimeMs > rfModifyTimeMs;
1225                                                                                        boolean changedValue = !rf.readLink().equals(df.readLink());
1226                                                                                        boolean changed = changedTime && changedValue;
1227                                                                                        if (changed) {
1228                                                                                                RemoteFile prf = rf.getParentFile();
1229                                                                                                if (prf != null && prf.exists()) {
1230                                                                                                        if (isDebug) {
1231                                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1232                                                                                                                                rf.getPath());
1233                                                                                                        }
1234                                                                                                        rf.forceDeleteLink(false);
1235                                                                                                        rf.createLink(df.readLink());
1236                                                                                                }
1237                                                                                        }
1238                                                                                } else {
1239                                                                                        RemoteFile prf = rf.getParentFile();
1240                                                                                        if (prf != null && prf.exists()) {
1241                                                                                                if (isDebug) {
1242                                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1243                                                                                                                        rf.getPath());
1244                                                                                                }
1245                                                                                                rf.forceDeleteLink(false);
1246                                                                                                rf.createLink(df.readLink());
1247                                                                                        }
1248                                                                                }
1249                                                                        } else {
1250                                                                                RemoteFile prf = rf.getParentFile();
1251                                                                                if (prf != null && prf.exists()) {
1252                                                                                        if (isDebug) {
1253                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1254                                                                                                                rf.getPath());
1255                                                                                        }
1256                                                                                        rf.createLink(df.readLink());
1257                                                                                }
1258                                                                        }
1259                                                                }
1260                                                        } else if (df.isFile()) {
1261                                                                if (!df.isLogicModify() && !df.isLogicAccess()) {
1262                                                                        if (rf.exists()) {
1263                                                                                if (rf.isFile()) {
1264                                                                                        long rfModifyTimeMs = rf.getModifyTime().getTime();
1265                                                                                        if (dfModifyTimeMs > rfModifyTimeMs) {
1266                                                                                                RemoteFile prf = rf.getParentFile();
1267                                                                                                if (prf != null && prf.exists()) {
1268                                                                                                        if (DiskFile.addQueuePathMapping(df.getPath())) {
1269                                                                                                                if (isDebug) {
1270                                                                                                                        LoggerFactory.getLogger(DiskFile.class)
1271                                                                                                                                        .mark("RemoteQueuing - {}", df.getPath());
1272                                                                                                                }
1273                                                                                                                df.logicAccess();
1274                                                                                                                rf.forceDeleteFile(false);
1275                                                                                                                df.writeToRemote(rf);
1276                                                                                                        } else {
1277                                                                                                                df.removeLogicAccess();
1278                                                                                                                df.removeLogicCheck();
1279                                                                                                        }
1280                                                                                                }
1281                                                                                        }
1282                                                                                } else {
1283                                                                                        rf.forceDeleteFile(false);
1284                                                                                }
1285                                                                        } else {
1286                                                                                RemoteFile prf = rf.getParentFile();
1287                                                                                if (prf != null && prf.exists()) {
1288                                                                                        if (DiskFile.addQueuePathMapping(df.getPath())) {
1289                                                                                                if (isDebug) {
1290                                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("RemoteQueuing - {}",
1291                                                                                                                        df.getPath());
1292                                                                                                }
1293                                                                                                df.logicAccess();
1294                                                                                                df.writeToRemote(rf);
1295                                                                                        } else {
1296                                                                                                df.removeLogicAccess();
1297                                                                                                df.removeLogicCheck();
1298                                                                                        }
1299                                                                                }
1300                                                                        }
1301                                                                }
1302                                                        } else if (df.isDir()) {
1303                                                                if (isStopSync() && isDebug) {
1304                                                                        LoggerFactory.getLogger(DiskFile.class).mark("Stopping queue");
1305                                                                }
1306                                                                if (!isStopSync()) {
1307                                                                        boolean allowedPoolSize = theFilter.getCacheArray().getUsingPoolSize() < theFilter
1308                                                                                        .getCacheArray().getMaxPoolSize() && checkUsage(rf);
1309                                                                        if (isDebug && allowedPoolSize) {
1310                                                                                LoggerFactory.getLogger(DiskFile.class).debug("Scaning - {}", df.getPath());
1311                                                                        }
1312                                                                        if (isDebug && !allowedPoolSize) {
1313                                                                                LoggerFactory.getLogger(DiskFile.class).debug("Skiped - {}", df.getPath());
1314                                                                        }
1315                                                                        if (rf.exists()) {
1316                                                                                if (rf.isDir()) {
1317                                                                                        if (isRoot || allowedPoolSize) {
1318                                                                                                df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1319                                                                                                                completedCallback);
1320                                                                                        }
1321                                                                                } else {
1322                                                                                        if (rf.isDir())
1323                                                                                                rf.forceDeleteDir(false);
1324
1325                                                                                        if (rf.isFile())
1326                                                                                                rf.forceDeleteFile(false);
1327
1328                                                                                        if (rf.isLink())
1329                                                                                                rf.forceDeleteLink(false);
1330
1331                                                                                        RemoteFile prf = rf.getParentFile();
1332                                                                                        if (isRoot || (prf != null && prf.exists())) {
1333                                                                                                if (isDebug) {
1334                                                                                                        LoggerFactory.getLogger(DiskFile.class)
1335                                                                                                                        .mark("ReCreateRemoteDir - {}", rf.getPath());
1336                                                                                                }
1337                                                                                                rf.mkdirs();
1338                                                                                                if (isRoot || allowedPoolSize) {
1339                                                                                                        df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1340                                                                                                                        completedCallback);
1341                                                                                                }
1342                                                                                        }
1343                                                                                }
1344                                                                        } else {
1345                                                                                RemoteFile prf = rf.getParentFile();
1346                                                                                if (isRoot || (prf != null && prf.exists())) {
1347                                                                                        if (isDebug) {
1348                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateRemoteDir - {}",
1349                                                                                                                rf.getPath());
1350                                                                                        }
1351                                                                                        rf.mkdirs();
1352                                                                                        if (isRoot || allowedPoolSize) {
1353                                                                                                df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1354                                                                                                                completedCallback);
1355                                                                                        }
1356                                                                                }
1357                                                                        }
1358                                                                        if (!isRoot && !allowedPoolSize) {
1359                                                                                df.removeLogicCheck();
1360                                                                        }
1361                                                                }
1362                                                        }
1363                                                }
1364                                        }
1365                                } catch (Exception e) {
1366                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1367                                        if (df != null) {
1368                                                df.removeLogicAccess();
1369                                        }
1370                                }
1371                        }
1372
1373                };
1374        }
1375
1376        public void setCopyStructureOnly(boolean copyStructureOnly) {
1377                this.copyStructureOnly = copyStructureOnly;
1378        }
1379
1380        public boolean isCopyStructureOnly() {
1381                return this.copyStructureOnly;
1382        }
1383
1384        private boolean isViEditSwp(String fileName) {
1385                return fileName.matches(TMP_MATCHES_EDIT) || fileName.matches(TMP_MATCHES_VI_SWP);
1386        }
1387
1388        protected static DiskFile copyAttrs(DiskFile source, boolean _copyStructureOnly, File _syncRoot) {
1389                source.setCopyStructureOnly(_copyStructureOnly);
1390                source.syncRoot = _syncRoot;
1391                return source;
1392        }
1393
1394        public File getParentFile() {
1395                return new File(getParent());
1396        }
1397
1398        protected boolean checkSyncPathAvailable(String path) {
1399                try {
1400                        return checkSyncPathAvailable(path, RemoteFile.SHOW_HIDDEN, RemoteFile.SHOW_LINK, RemoteFile.MAX_FILE_SZIE,
1401                                        SYNC_PATH_ALLOWED, SYNC_PATH_IGNORED);
1402                } catch (Exception e) {
1403                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1404                        return false;
1405                }
1406        }
1407
1408        protected static synchronized boolean addQueuePathMapping(String path) {
1409                Long addTimeMs = null;
1410                boolean allowed = checkQueuePathMapping();
1411
1412                if (!allowed)
1413                        return false;
1414
1415                addTimeMs = QUEUE_PATH_MAPPING.get(path);
1416
1417                if (addTimeMs == null) {
1418                        QUEUE_PATH_MAPPING.put(path, System.currentTimeMillis());
1419                        return true;
1420                }
1421
1422                return false;
1423        }
1424
1425        public static synchronized boolean checkQueuePathMapping() {
1426                Long addTimeMs = null;
1427                Map<String, Long> copyQueueMap = getQueuePathMapping();
1428                List<String> keyList = new ArrayList<String>(copyQueueMap.keySet());
1429                for (String key : keyList) {
1430                        addTimeMs = copyQueueMap.get(key);
1431                        if (addTimeMs != null) {
1432                                boolean timeout = (System.currentTimeMillis() - addTimeMs) >= LOGIC_TIMEOUT_MS;
1433
1434                                if (timeout) {
1435                                        QUEUE_PATH_MAPPING.remove(key);
1436                                }
1437                        }
1438                }
1439
1440                if (QUEUE_PATH_MAPPING.keySet().size() >= RemoteFile.MAX_QUEUE_SIZE)
1441                        return false;
1442
1443                return true;
1444        }
1445
1446        protected static synchronized void removeQueuePathMapping(String path) {
1447                QUEUE_PATH_MAPPING.remove(path);
1448        }
1449
1450        public static synchronized Map<String, Long> getQueuePathMapping() {
1451                return Collections.synchronizedMap(QUEUE_PATH_MAPPING);
1452        }
1453
1454        private static void debugLog(String point, RemoteFile rf, CacheArrayFilter filter) {
1455                debugLog(DiskFile.class, point, rf, filter);
1456        }
1457
1458}