Source code

001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import java.nio.file.Paths;
028import java.nio.file.Files;
029import java.io.IOException;
030import java.nio.file.OpenOption;
031import java.nio.file.StandardOpenOption;
032import java.nio.channels.FileChannel;
033import java.nio.channels.FileLock;
034import java.nio.file.Path;
035import java.nio.ByteBuffer;
036import com.killcoding.log.Logger;
037import java.net.URI;
038import java.util.Comparator;
039import java.util.List;
040import java.util.ArrayList;
041import java.nio.channels.SeekableByteChannel;
042import java.io.ByteArrayOutputStream;
043import java.nio.file.attribute.BasicFileAttributes;
044import java.nio.file.attribute.FileTime;
045import java.time.Instant;
046import java.util.Arrays;
047import java.io.ByteArrayInputStream;
048import java.sql.Timestamp;
049import java.util.stream.Stream;
050import java.util.stream.Collectors;
051import java.util.HashSet;
052import java.util.Set;
053import java.nio.file.DirectoryStream;
054import java.nio.file.CopyOption;
055import java.nio.file.LinkOption;
056import com.killcoding.tool.CommonTools;
057import com.killcoding.file.RemoteFile;
058import com.killcoding.log.LoggerFactory;
059import java.util.concurrent.Future;
060import com.killcoding.tool.ConfigProperties;
061import com.killcoding.datasource.Clock;
062import java.util.concurrent.Callable;
063import com.killcoding.file.DiskFile;
064import java.nio.file.attribute.FileAttribute;
065import java.text.SimpleDateFormat;
066import com.killcoding.cache.CacheArray;
067import com.killcoding.cache.CacheArrayFilter;
068import java.util.Map;
069import java.util.concurrent.ExecutorService;
070import java.util.concurrent.Executors;
071import java.nio.file.NoSuchFileException;
072import java.util.concurrent.ConcurrentHashMap;
073import java.util.Iterator;
074import java.util.Date;
075import java.time.ZonedDateTime;
076import java.time.LocalDateTime;
077import java.time.ZoneId;
078import java.time.format.DateTimeFormatter;
079import java.util.Calendar;
080import java.util.TimeZone;
081import java.text.ParseException;
082import java.util.Collections;
083import java.util.HashMap;
084import java.nio.file.FileAlreadyExistsException;
085import java.util.LinkedList;
086
087public class DiskFile extends BaseFile {
088    
089    public static final int RETRY_OPEN_LIMITED = 100;
090
091        protected static Integer MAX_POOL_SIZE = 100;
092        private static final Map<String, Long> QUEUE_PATH_MAPPING = new ConcurrentHashMap<String, Long>();
093
094        private static ExecutorService splitPool = null;
095
096        protected boolean copyStructureOnly = false;
097        private FileChannel channel = null;
098        private FileLock lock = null;
099        private long modifyTimeMs = 0L;
100
101        public DiskFile(String path) {
102                super(path);
103        }
104        
105        public DiskFile(File origin) {
106                super(origin);
107        }       
108
109        public static synchronized void initPool(int poolSize) {
110                if (splitPool == null) {
111                        MAX_POOL_SIZE = poolSize;
112                        LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
113                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
114                }
115        }
116
117        private static synchronized void initDefaultPool() {
118                if (splitPool == null) {
119                        LoggerFactory.getLogger(DiskFile.class).mark("MAX_POOL_SIZE={}", MAX_POOL_SIZE);
120                        splitPool = Executors.newFixedThreadPool(MAX_POOL_SIZE);
121                }
122        }
123
124        public void split(int partSize, FilePart filePart) throws IOException {
125                if (isLink()) {
126                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
127                }
128                if (isDir()) {
129                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
130                }
131                if (!exists()) {
132                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
133                }
134                initDefaultPool();
135                splitPool.execute(new Runnable() {
136                        @Override
137                        public void run() {
138                                SeekableByteChannel ch = null;
139                                int partIndex = -1;
140                                long fileSize = 0L;
141                                try {
142                                        Path originPath = Paths.get(origin.toURI());
143                                        Thread.currentThread().setName(String.format("DiskFile-split-%s", origin.getName()));
144                                        ch = Files.newByteChannel(originPath, StandardOpenOption.READ);
145                                        ByteBuffer bf = ByteBuffer.allocate(partSize);
146                                        while (ch.read(bf) != -1) {
147                                                bf.flip();
148                                                partIndex += 1;
149                                                List<Byte> parts = new ArrayList<Byte>();
150                                                while (bf.hasRemaining()) {
151                                                        parts.add(bf.get());
152                                                }
153                                                int partsSize = parts.size();
154                                                fileSize += partsSize;
155                                                byte[] part = new byte[partsSize];
156                                                for (int i = 0; i < partsSize; i++) {
157                                                        part[i] = parts.get(i);
158                                                }
159                                                filePart.process(partIndex, part);
160                                                bf.clear();
161                                        }
162                                        filePart.completed(partIndex, getModifyTimeForClock(), fileSize);
163                                } catch (Exception e) {
164                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
165                                } finally {
166                                        if (ch != null) {
167                                                try {
168                                                        ch.close();
169                                                } catch (IOException e) {
170                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
171                                                }
172                                        }
173                                        filePart.ended(partIndex, fileSize);
174                                }
175                        }
176                });
177        }
178
179        @Override
180        public void copyTo(String toPath) throws IOException {
181                Path originPath = Paths.get(origin.toURI());
182                Path destPath = Paths.get(toPath);
183                LoggerFactory.getLogger(DiskFile.class).debug("CopyTo: {} -> {}", originPath, destPath);
184                Files.copy(originPath, destPath);
185        }
186
187        @Override
188        public void moveTo(String toPath) throws IOException {
189                Path originPath = Paths.get(origin.toURI());
190                Path destPath = Paths.get(toPath);
191                LoggerFactory.getLogger(DiskFile.class).debug("MoveTo: {} -> {}", originPath, destPath);
192                if (Files.exists(destPath)) {
193                        Files.delete(destPath);
194                }
195                Files.move(originPath, destPath);
196        }
197
198        public RemoteFile writeToRemote(RemoteFile remoteFile) throws IOException {
199                return writeToRemote(remoteFile, new Runnable() {
200                        @Override
201                        public void run() {
202
203                        }
204                });
205        }
206
207        public RemoteFile writeToRemote(RemoteFile remoteFile, Runnable completedCallback) throws IOException {
208                if (remoteFile.exists(true)) {
209                        throw new IOException(String.format("The remote file '%s' already exists.", remoteFile.getPath()));
210                } else {
211                        remoteFile.copyFrom(copyStructureOnly, this);
212                        if (isDir()) {
213                                startFullAsync(0, this.getPath() + "/", remoteFile.getPath() + "/", completedCallback, null);
214                        }
215                }
216                return remoteFile;
217        }
218
219        protected boolean isLogicModify() throws Exception {
220                boolean b = false;
221                String tmpPath = toModifyTmpFilePath();
222
223                DiskFile df = null;
224                Path logicPath = Paths.get(tmpPath);
225                if (Files.exists(logicPath)) {
226                        df = new DiskFile(tmpPath);
227                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
228                        Timestamp dfModifyTime = df.getModifyTimeForClock();
229
230                        long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
231
232                        b = diffMs < LOGIC_TIMEOUT_MS;
233                }
234                return b;
235        }
236
237        protected DiskFile logicModify() throws Exception {
238                return logicModify(null);
239        }
240
241        protected DiskFile logicModify(String msg) throws Exception {
242                if (getParentFile().exists()) {
243                        String tmpLogicPath = toModifyTmpFilePath();
244
245                        if (tmpLogicPath == null)
246                                return null;
247
248                        Path logicPath = Paths.get(tmpLogicPath);
249                        DiskFile df = new DiskFile(tmpLogicPath);
250                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
251
252                        if (msg == null) {
253                                df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
254                                                false);
255                        } else {
256                                df.write(msg, false);
257                        }
258
259                        Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
260                        return df;
261                }
262                return null;
263        }
264
265        protected void removeLogicModify() {
266                try {
267                        String tmpPath = toModifyTmpFilePath();
268                        if (tmpPath != null) {
269                                DiskFile df = new DiskFile(tmpPath);
270                                DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
271                                if (df.exists()) {
272                                        df.delete();
273                                }
274                        }
275                } catch (Exception e) {
276                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
277                }
278        }
279
280        protected boolean isLogicAccess() throws Exception {
281            boolean b = false;
282                String tmpPath = toAccessTmpFilePath();
283                DiskFile df = null;
284                Path logicPath = Paths.get(tmpPath);
285                if (Files.exists(logicPath)) {
286                        df = new DiskFile(tmpPath);
287                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
288                        Timestamp dfModifyTime = df.getModifyTimeForClock();
289
290                        long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
291
292                        b = diffMs < LOGIC_ACCESS_TIMEOUT_MS;
293                }
294                return b;
295        }
296
297        protected DiskFile logicAccess() throws Exception {
298                return logicAccess(null);
299        }
300
301        protected DiskFile logicAccess(String msg) throws Exception {
302                if (getParentFile().exists()) {
303                        String tmpLogicPath = toAccessTmpFilePath();
304                        Path logicPath = Paths.get(tmpLogicPath);
305                        DiskFile df = new DiskFile(tmpLogicPath);
306                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
307                        if (msg == null) {
308                                df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
309                                                false);
310                        } else {
311                                df.write(msg, false);
312                        }
313
314                        Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
315                        return df;
316                }
317                return null;
318        }
319
320        protected void removeLogicAccess() {
321                try {
322                        String tmpPath = toAccessTmpFilePath();
323                        if (tmpPath != null) {
324                                DiskFile df = new DiskFile(tmpPath);
325                                DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
326                                if (df.exists()) {
327                                        df.delete();
328                                }
329                        }
330                } catch (Exception e) {
331                        LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
332                }
333        }
334
335        protected boolean isLogicCheck() throws Exception {
336                boolean b = false;
337                String tmpPath = toCheckTmpFilePath();
338                
339                DiskFile df = null;
340                Path logicPath = Paths.get(tmpPath);
341                if (Files.exists(logicPath)) {
342                        df = new DiskFile(tmpPath);
343                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
344                        Timestamp dfModifyTime = df.getModifyTimeForClock();
345                                
346                        long diffMs = Calendar.getInstance().getTimeInMillis() - dfModifyTime.getTime();
347
348                        b = diffMs < LOGIC_CHECK_TIMEOUT_MS;
349                }
350                return b;
351        }
352
353        protected DiskFile logicCheck() throws Exception {
354                return logicCheck(null);
355        }
356
357        public DiskFile logicCheck(String msg) throws Exception {
358        if (getParentFile().exists()) {
359                String tmpLogicPath = toCheckTmpFilePath();
360                if (tmpLogicPath == null)
361                        return null;
362    
363                Path logicPath = Paths.get(tmpLogicPath);
364                DiskFile df = new DiskFile(tmpLogicPath);
365                DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
366                if (msg == null) {
367                        df.write(String.format("%s-%s", Thread.currentThread().getName(), Thread.currentThread().getId()),
368                                        false);
369                } else {
370                        df.write(msg, false);
371                }
372    
373                Files.setLastModifiedTime(logicPath, FileTime.fromMillis(Calendar.getInstance().getTimeInMillis()));
374                return df;
375        }
376                return null;
377        }
378
379        protected void removeLogicCheck() {
380            try{
381                String tmpPath = toCheckTmpFilePath();
382                if (tmpPath != null) {
383                        DiskFile df = new DiskFile(tmpPath);
384                        DiskFile.copyAttrs(df, this.copyStructureOnly, this.syncRoot);
385                        if (df.exists()) {
386                                df.delete();
387                                DiskFile pdf = new DiskFile(this.getOrigin().getParent());
388                                DiskFile.copyAttrs(pdf, this.copyStructureOnly, this.syncRoot);
389                                if(pdf.isLogicCheck()){
390                                    pdf.removeLogicCheck();
391                                }
392                        }
393                }
394            }catch(Exception e){
395                LoggerFactory.getLogger(DiskFile.class).warn(e.getMessage(), e);
396            }
397        }
398
399        @Override
400        public boolean beforeDelete(boolean realDeleted) {
401                return true;
402        }
403
404        @Override
405        public void afterDelete(boolean realDeleted) {
406
407        }
408
409        @Override
410        public boolean beforeMkdirs() {
411                return true;
412        }
413
414        @Override
415        public void afterMkdirs() {
416
417        }
418
419        @Override
420        public boolean beforeCreateLink(String target) {
421                return true;
422        }
423
424        @Override
425        public void afterCreateLink(String target) {
426
427        }
428
429        @Override
430        public boolean beforeWrite() {
431                return true;
432        }
433
434        @Override
435        public void afterWrite() {
436
437        }
438
439        @Override
440        public boolean delete() throws IOException {
441                return deleteFrom();
442        }
443
444        private boolean deleteFrom() throws IOException {
445                boolean allowed = beforeDelete(true);
446                if (allowed) {
447                        LoggerFactory.getLogger(DiskFile.class).debug("Delete: {}", getPath());
448                        if (isFile()) {
449                                Path originPath = Paths.get(origin.toURI());
450                                boolean exists = Files.exists(originPath);
451                                if (exists) {
452                                        Files.deleteIfExists(Paths.get(origin.toURI()));
453                                        afterDelete(true);
454                                        return true;
455                                }
456                        }
457                        if (isDir()) {
458                                Path originPath = Paths.get(origin.toURI());
459                                boolean exists = Files.exists(originPath);
460                                if (exists) {
461                                        Files.walk(originPath).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
462                                        afterDelete(true);
463                                        return true;
464                                }
465                        }
466                        if (isLink()) {
467                                Path originPath = Paths.get(origin.toURI());
468                                boolean exists = Files.exists(originPath);
469                                if (exists) {
470                                        Files.deleteIfExists(Paths.get(origin.toURI()));
471                                        afterDelete(true);
472                                        return true;
473                                }
474                        }
475                }
476                return false;
477        }
478
479        @Override
480        public boolean isFile() throws IOException {
481                return this.origin.isFile() && !isLink();
482        }
483
484        @Override
485        public boolean isDirectory() throws IOException {
486                return this.origin.isDirectory() && !isLink();
487        }
488
489        @Override
490        public boolean isLink() throws IOException {
491                return Files.isSymbolicLink(Paths.get(this.origin.toURI()));
492        }
493
494        @Override
495        public boolean exists() throws IOException {
496                return this.origin.exists();
497        }
498
499        @Override
500        public boolean mkdirs() throws IOException {
501                boolean allowed = beforeMkdirs();
502                if (allowed) {
503                        LoggerFactory.getLogger(DiskFile.class).debug("Mkidrs: {}", getPath());
504                        boolean b = this.origin.mkdirs();
505                        if (b && modifyTimeMs > 0) {
506                                setModifyTime(new Timestamp(modifyTimeMs));
507                        }
508                        afterMkdirs();
509                        return b;
510                }
511                return false;
512        }
513
514        @Override
515        public boolean write(byte[] data, boolean append) throws IOException {
516
517                boolean allowed = beforeWrite();
518
519                if (!allowed)
520                        return false;
521
522                if (data == null)
523                        data = new byte[] {};
524
525                try {
526
527                        OpenOption[] options = null;
528                        if (append) {
529                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
530                                                StandardOpenOption.APPEND };
531                        } else {
532                                options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
533                                                StandardOpenOption.TRUNCATE_EXISTING };
534                        }
535
536                        File parent = origin.getParentFile();
537                        if (!parent.exists()) {
538                                parent.mkdirs();
539                        }
540                        int retryCount = 0;
541                        Path path = Paths.get(origin.toURI());
542                        while(retryCount < RETRY_OPEN_LIMITED){
543                            retryCount++;
544                            try{
545                            channel = FileChannel.open(path, options);
546                            if(channel != null){
547                                lock = channel.lock();
548                                if (lock != null) {
549                                        ByteBuffer buf = ByteBuffer.wrap(data);
550                                        channel.write(buf);
551                                        if (modifyTimeMs > 0) {
552                                                setModifyTime(new Timestamp(modifyTimeMs));
553                                        }
554                                        return true;
555                                }
556                            }
557                            }catch(Exception e){
558                                LoggerFactory.getLogger(DiskFile.class).debug(e.getMessage(),e);
559                                
560                                if(retryCount >= RETRY_OPEN_LIMITED) throw new IOException(e.getMessage(),e);
561                                
562                            }
563                        }
564                        return false;
565                } catch (Exception e) {
566                        throw e;
567                } finally {
568                        close();
569                }
570        }
571
572        public boolean manualOpen(boolean append) throws IOException, FileAlreadyExistsException {
573
574                boolean allowed = beforeWrite();
575
576                if (!allowed)
577                        return false;
578
579                OpenOption[] options = null;
580                if (append) {
581                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
582                                        StandardOpenOption.APPEND };
583                } else {
584                        options = new OpenOption[] { StandardOpenOption.CREATE, StandardOpenOption.WRITE,
585                                        StandardOpenOption.TRUNCATE_EXISTING };
586                }
587                Path path = Paths.get(origin.toURI());
588                File parent = origin.getParentFile();
589                if (parent != null && !parent.exists() && !parent.mkdirs()) {
590                        throw new IOException(String.format("Failed to create parent directory: %s", parent));
591                }
592                if (!Files.exists(path)) {
593                        try {
594                                Files.createFile(path);
595                        } catch (FileAlreadyExistsException e) {
596                                throw e;
597                        }
598                }
599                int retryCount = 0;
600                while(retryCount < RETRY_OPEN_LIMITED){
601                    retryCount++;
602                    try{
603                        channel = FileChannel.open(path, options);
604                        lock = channel.lock();
605                        return lock != null;
606                    }catch(Exception e){
607                        LoggerFactory.getLogger(DiskFile.class).debug(e.getMessage(),e);
608                        
609                        if(retryCount >= RETRY_OPEN_LIMITED) throw new IOException(e.getMessage(),e);
610                    }
611                }
612                return false;
613        }
614
615        public void manualWrite(byte[] data) throws IOException {
616                if (data == null)
617                        data = new byte[] {};
618
619                if (lock != null) {
620                        ByteBuffer buf = ByteBuffer.wrap(data);
621                        channel.write(buf);
622                        if (modifyTimeMs > 0) {
623                                setModifyTime(new Timestamp(modifyTimeMs));
624                        }
625                }
626        }
627
628        public void manualWrite(String data) throws IOException {
629                manualWrite(data.getBytes(CHARSET));
630        }
631
632        public void manualClose() throws IOException {
633                close();
634        }
635
636        @Override
637        public boolean write(byte[] data) throws IOException {
638                return write(data, false);
639        }
640
641        @Override
642        public boolean write(String data, boolean append) throws IOException {
643                return write(data.getBytes(CHARSET), append);
644        }
645
646        public boolean write(String data, String charset, boolean append) throws IOException {
647                return write(data.getBytes(charset), append);
648        }
649
650        @Override
651        public boolean write(String data) throws IOException {
652                return write(data.getBytes(CHARSET));
653        }
654
655        public boolean write(String data, String charset) throws IOException {
656                return write(data.getBytes(charset));
657        }
658
659        @Override
660        public boolean createLink(String target) throws IOException {
661                boolean allowed = beforeCreateLink(target);
662                if (allowed) {
663                        LoggerFactory.getLogger(DiskFile.class).debug("CreateLink: {}", getPath());
664                        try {
665                                Path link = Paths.get(origin.toURI());
666                                Path targetPath = Paths.get(target);
667                                Files.createSymbolicLink(link, targetPath);
668                                if (modifyTimeMs > 0) {
669                                        setModifyTime(new Timestamp(modifyTimeMs));
670                                }
671                                afterCreateLink(target);
672                                return true;
673                        } catch (IOException e) {
674                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
675                        }
676                }
677                return false;
678        }
679
680        @Override
681        public String readLink() throws IOException {
682                Path link = Paths.get(origin.toURI());
683                return Files.readSymbolicLink(link).toString();
684        }
685
686        @Override
687        public byte[] readAllBytes() throws IOException {
688                return readAllBytesFrom();
689        }
690
691        private byte[] readAllBytesFrom() throws IOException {
692                if (isLink()) {
693                        throw new IOException(String.format("The disk file '%s' is a link.", origin.getAbsolutePath()));
694                }
695                if (isDir()) {
696                        throw new IOException(String.format("The disk file '%s' is a folder.", origin.getAbsolutePath()));
697                }
698                if (!exists()) {
699                        throw new IOException(String.format("The disk file '%s' does not exist.", origin.getAbsolutePath()));
700                }
701                return Files.readAllBytes(Paths.get(getPath()));
702        }
703
704        @Override
705        public String readAllString() throws IOException {
706                return readAllString(BaseFile.CHARSET);
707        }
708
709        public String readAllString(String charset) throws IOException {
710                ByteArrayOutputStream baos = null;
711                try {
712                        byte[] bytes = readAllBytes();
713                        baos = new ByteArrayOutputStream();
714                        baos.write(bytes);
715                        baos.flush();
716                        return baos.toString(charset);
717                } finally {
718                        if (baos != null) {
719                                try {
720                                        baos.close();
721                                } catch (IOException e) {
722                                        Logger.systemError(DiskFile.class, e.getMessage(), e);
723                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
724                                }
725                        }
726                }
727        }
728
729        @Override
730        public long size() throws IOException {
731                if (isFile()) {
732                        Path path = Paths.get(origin.getAbsolutePath());
733                        return Files.size(path);
734                }
735                return 0L;
736        }
737
738        private void close() {
739                if (lock != null) {
740                        try {
741                                lock.release();
742                        } catch (IOException e) {
743                                Logger.systemError(DiskFile.class, e.getMessage(), e);
744                        }
745                        try {
746                                lock.close();
747                        } catch (IOException e) {
748                                Logger.systemError(DiskFile.class, e.getMessage(), e);
749                        } finally {
750                                lock = null;
751                        }
752                }
753
754                if (channel != null) {
755                        try {
756                                channel.close();
757                        } catch (IOException e) {
758                                Logger.systemError(DiskFile.class, e.getMessage(), e);
759                        } finally {
760                                channel = null;
761                        }
762                }
763                afterWrite();
764        }
765
766        @Override
767        public boolean complete() throws IOException {
768                close();
769                return true;
770        }
771
772        public void setModifyTimeMs(long modifyTimeMs) throws IOException {
773                this.modifyTimeMs = modifyTimeMs;
774        }
775
776        public void setModifyTimeMsFromClock(long modifyTimeMsFromClock) throws IOException {
777                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
778                String localTzId = Calendar.getInstance().getTimeZone().getID();
779                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, new Timestamp(modifyTimeMsFromClock));
780                this.modifyTimeMs = fileLocalTimestamp.getTime();
781        }
782
783        @Override
784        public void setModifyTime(Timestamp modifyTime) throws IOException {
785                if (exists()) {
786                        Path originPath = Paths.get(origin.toURI());
787                        Files.setLastModifiedTime(originPath, FileTime.fromMillis(modifyTime.getTime()));
788                }
789        }
790
791        public void setModifyTimeFromClock(Timestamp modifyTimeFromClock) throws IOException {
792                String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
793                String localTzId = Calendar.getInstance().getTimeZone().getID();
794                Timestamp fileLocalTimestamp = timeZoneConver(localTzId, remoteTzId, modifyTimeFromClock);
795                setModifyTime(fileLocalTimestamp);
796        }
797
798        @Override
799        public Timestamp getModifyTime() throws IOException {
800                Path originPath = Paths.get(origin.toURI());
801                if (Files.exists(originPath)) {
802                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
803                        FileTime fileTime = attr.lastModifiedTime();
804                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
805                        return fileLocalTimestamp;
806                }
807                return null;
808        }
809
810        public Timestamp getModifyTimeForClock() throws IOException {
811                Path originPath = Paths.get(origin.toURI());
812                if (Files.exists(originPath)) {
813                        BasicFileAttributes attr = Files.readAttributes(originPath, BasicFileAttributes.class);
814                        FileTime fileTime = attr.lastModifiedTime();
815                        Timestamp fileLocalTimestamp = new Timestamp(fileTime.toMillis());
816                        String remoteTzId = new Clock().getCalendar().getTimeZone().getID();
817                        String localTzId = Calendar.getInstance().getTimeZone().getID();
818                        Timestamp fileRemoteTimestamp = timeZoneConver(remoteTzId, localTzId, fileLocalTimestamp);
819                        return fileRemoteTimestamp;
820                }
821                return null;
822        }
823
824        public void startFullAsync(long timerMs, String replaceTo) throws IOException {
825                startFullAsync(timerMs, getPath() + "/", replaceTo, null, null);
826        }
827
828        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback) throws IOException {
829                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, null);
830        }
831
832        public void startFullAsync(long timerMs, String replaceTo, Runnable completedCallback,
833                        Runnable checkDeleteCompletedCallback) throws IOException {
834                startFullAsync(timerMs, getPath() + "/", replaceTo, completedCallback, checkDeleteCompletedCallback);
835        }
836
837        private void startFullAsync(long timerMs, String replaceRootPath, String replaceTo, Runnable completedCallback,
838                        Runnable checkDeleteCompletedCallback) throws IOException {
839
840                if (!RemoteFile.FIRST_LOADED) {
841                        RemoteFile firstLoaded = new RemoteFile(replaceTo);
842                }
843
844                Runnable runnable = new Runnable() {
845                        @Override
846                        public void run() {
847                                try {
848                                        startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
849                                } catch (Exception e) {
850                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
851                                }
852                        }
853                };
854                Executors.newFixedThreadPool(1).execute(runnable);
855
856                runnable = new Runnable() {
857                        @Override
858                        public void run() {
859                                try {
860                                        startFullSyncForDelete(-1, timerMs, replaceTo, replaceRootPath, checkDeleteCompletedCallback);
861                                } catch (Exception e) {
862                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
863                                }
864                        }
865                };
866                Executors.newFixedThreadPool(1).execute(runnable);
867        }
868
869        private synchronized void startFullSyncForDelete(final Integer filePartDataTable, final long timerMs,
870                        final String replaceTo, final String replaceRootPath, final Runnable completedCallback) throws IOException {
871                DiskFile the = this;
872                RemoteFile rootDir = null;
873                if (!CommonTools.isBlank(replaceTo)) {
874                        rootDir = new RemoteFile(replaceTo);
875                }
876                if (rootDir == null) {
877                        throw new IOException("The root folder is null.");
878                }
879                RemoteFile _rootDir = rootDir;
880                Integer maxDataTable = rootDir.getMaxFilePartDataTable();
881
882                Integer _filePartDataTable = filePartDataTable;
883
884                if (filePartDataTable > maxDataTable)
885                        _filePartDataTable = -1;
886
887                final Integer finalFilePartDataTable = _filePartDataTable;
888
889                final CacheArray rows = new CacheArray();
890
891                rows.filter(getScanDeleteFilter(rootDir, finalFilePartDataTable, timerMs, replaceTo, replaceRootPath,
892                                completedCallback));
893
894                Runnable runnable = new Runnable() {
895                        @Override
896                        public void run() {
897                                try {
898                                        _rootDir.listAllForScanDelete(finalFilePartDataTable, rows);
899                                } catch (Exception e) {
900                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
901                                }
902                        }
903                };
904                Executors.newFixedThreadPool(1).execute(runnable);
905        }
906
907        private CacheArrayFilter getScanDeleteFilter(final RemoteFile rootDir, final Integer filePartDataTable,
908                        final long timerMs, final String replaceTo, final String replaceRootPath,
909                        final Runnable completedCallback) {
910                DiskFile the = this;
911                CacheArrayFilter filter = new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
912                        @Override
913                        public void execute(Integer index, Object o) {
914
915                                try {
916                                        Map<String, Object> item = (Map<String, Object>) o;
917                                        Object id = item.get("id");
918                                        String parent = (String) item.get("file_parent_path");
919                                        String fileName = (String) item.get("file_name");
920                                        String pf = String.format("%s/%s", parent, fileName);
921                                        RemoteFile rf = new RemoteFile(pf);
922                                        Thread.currentThread()
923                                                        .setName(String.format("DiskFile-startFullSyncForDelete-(DataTable=%s Index=%s)-%s",
924                                                                        filePartDataTable, index, fileName));
925
926                                        if (memoryUsage() < RemoteFile.MAX_MEMORY_USAGE) {
927                                                String toDf = pf;
928                                                if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
929                                                        toDf = pf.replaceFirst(replaceTo + "[/]*", replaceRootPath + "/");
930                                                }
931
932                                                DiskFile df = new DiskFile(toDf);
933                                                DiskFile.copyAttrs(df, the.copyStructureOnly, the.syncRoot);
934                                                boolean isDebug = isDebug(rf);
935                                                boolean isRoot = rootDir.getPath().equals(rf.getPath());
936                                                boolean isSyncedHost = rf.isSyncedOnHostname(id);
937                                                boolean rfExists = rf.exists();
938
939                                                boolean allowedForceDelete = rootDir.exists() && !isRoot && isSyncedHost && rfExists
940                                                                && !df.exists() && !df.isLogicModify();
941
942                                                if (allowedForceDelete) {
943                                                        df.logicModify();
944                                                        if (isDebug) {
945                                                                LoggerFactory.getLogger(DiskFile.class).mark("ForceRemoteDelete - {}", rf.getPath());
946                                                        }
947                                                        rf.forceDeleteAll(true);
948                                                        df.removeLogicModify();
949                                                }
950                                        } else {
951                                                LoggerFactory.getLogger(DiskFile.class).warn("Over max memory usage 'MAX_MEMORY_USAGE={}'.",
952                                                                RemoteFile.MAX_MEMORY_USAGE);
953                                        }
954                                } catch (Exception e) {
955                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
956                                }
957                        }
958
959                        @Override
960                        public void completed(Integer size) {
961                                if (isStopSync()) {
962                                        LoggerFactory.getLogger(DiskFile.class).mark("Stoped start full sync for delete.");
963                                }
964                                if (timerMs <= 0) {
965                                        LoggerFactory.getLogger(DiskFile.class).mark("Completed start full sync for delete.");
966                                }
967
968                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
969
970                                if (isRootCompleted && completedCallback != null) {
971                                        completedCallback.run();
972                                }
973
974                                if (!isStopSync() && timerMs > 0) {
975                                        while (true) {
976
977                                                if (isStopSync())
978                                                        break;
979
980                                                try {
981                                                        Thread.sleep(timerMs);
982                                                } catch (InterruptedException e) {
983                                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
984                                                }
985                                                boolean allowed = checkUsage(rootDir);
986                                                if (allowed) {
987                                                        try {
988                                                                if (!isStopSync()) {
989                                                                        startFullSyncForDelete(filePartDataTable + 1, timerMs, replaceTo, replaceRootPath,
990                                                                                        completedCallback);
991                                                                }
992                                                        } catch (IOException e) {
993                                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
994                                                        }
995                                                        break;
996                                                }
997                                        }
998                                }
999                        }
1000                };
1001                return filter;
1002        }
1003
1004        private void startFullAsyncThread(final long timerMs, String replaceRootPath, String replaceTo,
1005                        final Runnable completedCallback) {
1006                stopSync = false;
1007                this.setCopyStructureOnly(RemoteFile.COPY_STRUCTURE_ONLY);
1008                final DiskFile the = this;
1009                Thread.currentThread().setName(String.format("DiskFile-startFullAsyncThread-%s", the.getOrigin().getName()));
1010                replaceRootPath = replacePath(replaceRootPath);
1011                replaceTo = replacePath(replaceTo);
1012                DiskFile rootDisk = new DiskFile(replaceRootPath);
1013                boolean rootExists = false;
1014                try {
1015                        rootExists = rootDisk.exists();
1016                } catch (Exception e) {
1017                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1018                }
1019                if (!rootExists) {
1020                        Logger.systemError(DiskFile.class, "The root path '{}' does not exist.", rootDisk.getPath());
1021                }
1022                boolean isRoot = the.getPath().equals(rootDisk.getPath());
1023                LoggerFactory.getLogger(DiskFile.class).debug("IsRoot: {} - {}", isRoot, the.getPath());
1024                LoggerFactory.getLogger(DiskFile.class).debug("CacheArrayUsage: {}", CacheArray.getUsage());
1025                if (rootExists) {
1026
1027                        DirectoryStream<Path> stream = null;
1028                        try {
1029                                if (syncRoot == null) {
1030                                        File replaceRootFile = new File(replaceRootPath);
1031                                        if (replaceRootFile.getParent() == null) {
1032                                                throw new IOException("Cannot sync the root path.");
1033                                        }
1034                                        String replaceRootParent = replacePath(replaceRootFile.getParent());
1035                                        syncRoot = new File(String.format("%s/%s", replaceRootParent, replaceRootFile.getName()));
1036                                }
1037
1038                                if (!CommonTools.isBlank(replaceTo)) {
1039                                        RemoteFile rootDir = new RemoteFile(replaceTo);
1040                                        if (!rootDir.exists()) {
1041                                                rootDir.setModifyTime(this.getModifyTimeForClock());
1042                                                rootDir.mkdirs();
1043                                        }
1044                                }
1045                                Path originPath = Paths.get(origin.toURI());
1046                                CacheArray rows = new CacheArray();
1047                                CacheArrayFilter filter = getFullAsyncFilter(the, timerMs, replaceRootPath, replaceTo,
1048                                                completedCallback);
1049                                rows.filter(filter);
1050                                stream = Files.newDirectoryStream(originPath);
1051                                for (Path p : stream) {
1052                                        if (p != null) {
1053                                                File f = p.toFile();
1054                                                boolean isFile = f.isFile();
1055                                                if (isFile) {
1056                                                        boolean isWritingDf = f.getName().matches(TMP_WRITING_SWP);
1057                                                        if (isWritingDf) {
1058                                                                BasicFileAttributes attributes = Files.readAttributes(p, BasicFileAttributes.class);
1059                                                                FileTime creationTime = attributes.creationTime();
1060                                                                boolean timeouted = (Calendar.getInstance().getTimeInMillis()
1061                                                                                - creationTime.toMillis()) > (RemoteFile.BATCH_SIZE * LOGIC_TIMEOUT_MS);
1062                                                                if (timeouted) {
1063                                                                        try {
1064                                                                                boolean exists = Files.exists(p);
1065                                                                                if (exists) {
1066                                                                                        Files.deleteIfExists(p);
1067                                                                                }
1068                                                                        } catch (Exception e) {
1069                                                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1070                                                                        }
1071                                                                }
1072                                                        }
1073                                                }
1074                                                boolean allowedPath = the.checkSyncPathAvailable(p.toString());
1075                                                if (!allowedPath) {
1076                                                        LoggerFactory.getLogger(DiskFile.class).debug("Ignored: {}", p.toString());
1077                                                }
1078                                                if (allowedPath) {
1079                                                        rows.add(p);
1080                                                }
1081                                        }
1082                                }
1083                                rows.add(null);
1084                        } catch (Exception e) {
1085                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1086                                removeLogicAccess();
1087                                if (!(e instanceof NoSuchFileException)) {
1088                                        if (!isStopSync() && timerMs > 0) {
1089                                                try {
1090                                                        Thread.sleep(timerMs);
1091                                                } catch (InterruptedException ee) {
1092                                                        LoggerFactory.getLogger(DiskFile.class).error(ee.getMessage(), ee);
1093                                                }
1094                                                if (!isStopSync()) {
1095                                                        startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
1096                                                }
1097                                        }
1098                                }
1099                        } finally {
1100                                if (stream != null) {
1101                                        try {
1102                                                stream.close();
1103                                        } catch (IOException e) {
1104                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1105                                        }
1106                                }
1107                        }
1108                }
1109        }
1110
1111        private CacheArrayFilter getFullAsyncFilter(DiskFile the, long timerMs, String replaceRootPath, String replaceTo,
1112                        Runnable completedCallback) {
1113
1114                return new CacheArrayFilter(CACHE_ARRAY_FILTER_TIMER) {
1115                        RemoteFile rf = null;
1116                        final CacheArrayFilter theFilter = this;
1117
1118                        @Override
1119                        public void completed(Integer size) {
1120                                boolean isRootCompleted = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1121
1122                                if (isRootCompleted) {
1123                                        try {
1124                                                if (rf != null && the.isStopSync() && isDebug(rf)) {
1125                                                        LoggerFactory.getLogger(DiskFile.class).mark("StopedRootScan - {}", the.getPath());
1126                                                }
1127                                                if (!the.isStopSync() && timerMs > 0) {
1128                                                        Thread.sleep(timerMs);
1129                                                        if (!the.isStopSync()) {
1130                                                                if (rf != null && isDebug(rf)) {
1131                                                                        LoggerFactory.getLogger(DiskFile.class).mark("RootScaning - {}", the.getPath());
1132                                                                }
1133                                                                the.startFullAsyncThread(timerMs, replaceRootPath, replaceTo, completedCallback);
1134                                                        }
1135                                                }
1136                                                if (completedCallback != null) {
1137                                                        completedCallback.run();
1138                                                }
1139                                        } catch (Exception e) {
1140                                                LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1141                                        }
1142                                }
1143                        }
1144
1145                        @Override
1146                        public void execute(Integer index, Object o) {
1147                                DiskFile df = null;
1148                                try {
1149                                        Path p = (Path) o;
1150                                        String pf = replacePath(p.toFile().getAbsolutePath());
1151                                        Thread.currentThread().setName(
1152                                                        String.format("DiskFile-getFullAsyncFilter-%s", p.toFile().getParentFile().getName()));
1153                                        String toRf = pf;
1154                                        if (!CommonTools.isBlank(replaceRootPath) && !CommonTools.isBlank(replaceTo)) {
1155                                                toRf = pf.replaceFirst(replaceRootPath + "[/]*", replaceTo + "/");
1156                                        }
1157
1158                                        df = new DiskFile(pf);
1159                                        DiskFile.copyAttrs(df, the.copyStructureOnly, the.syncRoot);
1160                                        rf = new RemoteFile(toRf);
1161
1162                                        boolean isRoot = the.getPath().equals(new DiskFile(replaceRootPath).getPath());
1163
1164                                        boolean allowedConn = checkUsage(rf);
1165                                        String fileName = p.toFile().getName();
1166                                        boolean isDebug = isDebug(rf);
1167
1168                                        boolean timeouted = rf.isTimeout();
1169                                        if (timeouted && RemoteFile.addQueuePathMapping(rf.getPath()) && allowedConn) {
1170                                                if (isDebug) {
1171                                                        LoggerFactory.getLogger(DiskFile.class).mark("ForceDelete(Timeouted) - {}", rf.getPath());
1172                                                }
1173                                                rf.forceDeleteFile(false);
1174                                                RemoteFile.removeQueuePathMapping(rf.getPath());
1175                                        }
1176                                        Timestamp modifyTime = df.getModifyTimeForClock();
1177                                        boolean allowedModifyTime = modifyTime != null;
1178                                        boolean allowed = DiskFile.checkQueuePathMapping() && !isStopSync() && allowedModifyTime
1179                                                        && allowedConn;
1180
1181                                        if (isDebug) {
1182                                                debugLog("CheckUsage", rf, this);
1183                                        }
1184
1185                                        if (!isRoot && allowed) {
1186                                                allowed = !df.isLogicCheck();
1187                                        }
1188
1189                                        if (allowed) {
1190
1191                                                if (!isRoot)
1192                                                        df.logicCheck();
1193
1194                                                rf.setModifyTime(modifyTime);
1195                                                long dfModifyTimeMs = modifyTime.getTime();
1196                                                boolean blocked = false;
1197                                                boolean isRealDelete = rf.isLastOperateRealDelete();
1198
1199                                                if (isRealDelete) {
1200                                                        String clientHostname = CommonTools.getHostname();
1201                                                        String delHostname = rf.getLastSourceHostname();
1202                                                        boolean isOwnerDel = clientHostname.equals(delHostname);
1203                                                        if (isOwnerDel) {
1204                                                                Timestamp deletedTime = rf.getLastOperateTime();
1205                                                                if (deletedTime != null) {
1206                                                                        boolean waiting = new Clock().getTime() - deletedTime.getTime() <= LOGIC_TIMEOUT_MS;
1207                                                                        if (waiting) {
1208                                                                                blocked = true;
1209                                                                        }
1210                                                                }
1211                                                        } else {
1212                                                                Timestamp deletedTime = rf.getLastOperateTime();
1213                                                                if (deletedTime != null) {
1214                                                                        boolean olded = modifyTime.getTime() < deletedTime.getTime();
1215                                                                        if (olded) {
1216                                                                                blocked = true;
1217                                                                        }
1218                                                                }
1219                                                        }
1220                                                }
1221
1222                                                boolean haveParent = rf.getParentFile().exists();
1223                                                if (!blocked && haveParent) {
1224                                                        if (df.isLink()) {
1225                                                                if (!df.isLogicAccess()) {
1226                                                                        df.logicAccess();
1227                                                                        if (rf.exists()) {
1228                                                                                if (rf.isLink()) {
1229                                                                                        long rfModifyTimeMs = rf.getModifyTime().getTime();
1230                                                                                        boolean changedTime = dfModifyTimeMs > rfModifyTimeMs;
1231                                                                                        boolean changedValue = !rf.readLink().equals(df.readLink());
1232                                                                                        boolean changed = changedTime && changedValue;
1233                                                                                        if (changed) {
1234                                                                                                RemoteFile prf = rf.getParentFile();
1235                                                                                                if (prf != null && prf.exists()) {
1236                                                                                                        if (isDebug) {
1237                                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1238                                                                                                                                rf.getPath());
1239                                                                                                        }
1240                                                                                                        rf.forceDeleteLink(false);
1241                                                                                                        rf.createLink(df.readLink());
1242                                                                                                }
1243                                                                                        }
1244                                                                                } else {
1245                                                                                        RemoteFile prf = rf.getParentFile();
1246                                                                                        if (prf != null && prf.exists()) {
1247                                                                                                if (isDebug) {
1248                                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1249                                                                                                                        rf.getPath());
1250                                                                                                }
1251                                                                                                rf.forceDeleteLink(false);
1252                                                                                                rf.createLink(df.readLink());
1253                                                                                        }
1254                                                                                }
1255                                                                        } else {
1256                                                                                RemoteFile prf = rf.getParentFile();
1257                                                                                if (prf != null && prf.exists()) {
1258                                                                                        if (isDebug) {
1259                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateLink - {}",
1260                                                                                                                rf.getPath());
1261                                                                                        }
1262                                                                                        rf.createLink(df.readLink());
1263                                                                                }
1264                                                                        }
1265                                                                }
1266                                                        } else if (df.isFile()) {
1267                                                                if (!df.isLogicModify() && !df.isLogicAccess()) {
1268                                                                        if (rf.exists()) {
1269                                                                                if (rf.isFile()) {
1270                                                                                        long rfModifyTimeMs = rf.getModifyTime().getTime();
1271                                                                                        if (dfModifyTimeMs > rfModifyTimeMs) {
1272                                                                                                RemoteFile prf = rf.getParentFile();
1273                                                                                                if (prf != null && prf.exists()) {
1274                                                                                                        if (DiskFile.addQueuePathMapping(df.getPath())) {
1275                                                                                                                if (isDebug) {
1276                                                                                                                        LoggerFactory.getLogger(DiskFile.class)
1277                                                                                                                                        .mark("RemoteQueuing - {}", df.getPath());
1278                                                                                                                }
1279                                                                                                                df.logicAccess();
1280                                                                                                                rf.forceDeleteFile(false);
1281                                                                                                                df.writeToRemote(rf);
1282                                                                                                        } else {
1283                                                                                                                df.removeLogicAccess();
1284                                                                                                                df.removeLogicCheck();
1285                                                                                                        }
1286                                                                                                }
1287                                                                                        }
1288                                                                                } else {
1289                                                                                        rf.forceDeleteFile(false);
1290                                                                                }
1291                                                                        } else {
1292                                                                                RemoteFile prf = rf.getParentFile();
1293                                                                                if (prf != null && prf.exists()) {
1294                                                                                        if (DiskFile.addQueuePathMapping(df.getPath())) {
1295                                                                                                if (isDebug) {
1296                                                                                                        LoggerFactory.getLogger(DiskFile.class).mark("RemoteQueuing - {}",
1297                                                                                                                        df.getPath());
1298                                                                                                }
1299                                                                                                df.logicAccess();
1300                                                                                                df.writeToRemote(rf);
1301                                                                                        } else {
1302                                                                                                df.removeLogicAccess();
1303                                                                                                df.removeLogicCheck();
1304                                                                                        }
1305                                                                                }
1306                                                                        }
1307                                                                }
1308                                                        } else if (df.isDir()) {
1309                                                                if (isStopSync() && isDebug) {
1310                                                                        LoggerFactory.getLogger(DiskFile.class).mark("Stopping queue");
1311                                                                }
1312                                                                if (!isStopSync()) {
1313                                                                        boolean allowedPoolSize = theFilter.getCacheArray().getUsingPoolSize() < theFilter
1314                                                                                        .getCacheArray().getMaxPoolSize() && checkUsage(rf);
1315                                                                        if (isDebug && allowedPoolSize) {
1316                                                                                LoggerFactory.getLogger(DiskFile.class).debug("Scaning - {}", df.getPath());
1317                                                                        }
1318                                                                        if (isDebug && !allowedPoolSize) {
1319                                                                                LoggerFactory.getLogger(DiskFile.class).debug("Skiped - {}", df.getPath());
1320                                                                        }
1321                                                                        if (rf.exists()) {
1322                                                                                if (rf.isDir()) {
1323                                                                                        if (isRoot || allowedPoolSize) {
1324                                                                                                df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1325                                                                                                                completedCallback);
1326                                                                                        }
1327                                                                                } else {
1328                                                                                        if (rf.isDir())
1329                                                                                                rf.forceDeleteDir(false);
1330
1331                                                                                        if (rf.isFile())
1332                                                                                                rf.forceDeleteFile(false);
1333
1334                                                                                        if (rf.isLink())
1335                                                                                                rf.forceDeleteLink(false);
1336
1337                                                                                        RemoteFile prf = rf.getParentFile();
1338                                                                                        if (isRoot || (prf != null && prf.exists())) {
1339                                                                                                if (isDebug) {
1340                                                                                                        LoggerFactory.getLogger(DiskFile.class)
1341                                                                                                                        .mark("ReCreateRemoteDir - {}", rf.getPath());
1342                                                                                                }
1343                                                                                                rf.mkdirs();
1344                                                                                                if (isRoot || allowedPoolSize) {
1345                                                                                                        df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1346                                                                                                                        completedCallback);
1347                                                                                                }
1348                                                                                        }
1349                                                                                }
1350                                                                        } else {
1351                                                                                RemoteFile prf = rf.getParentFile();
1352                                                                                if (isRoot || (prf != null && prf.exists())) {
1353                                                                                        if (isDebug) {
1354                                                                                                LoggerFactory.getLogger(DiskFile.class).mark("CreateRemoteDir - {}",
1355                                                                                                                rf.getPath());
1356                                                                                        }
1357                                                                                        rf.mkdirs();
1358                                                                                        if (isRoot || allowedPoolSize) {
1359                                                                                                df.startFullAsyncThread(timerMs, replaceRootPath, replaceTo,
1360                                                                                                                completedCallback);
1361                                                                                        }
1362                                                                                }
1363                                                                        }
1364                                                                        if (!isRoot && !allowedPoolSize) {
1365                                                                                df.removeLogicCheck();
1366                                                                        }
1367                                                                }
1368                                                        }
1369                                                }
1370                                        }
1371                                } catch (Exception e) {
1372                                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1373                                        if (df != null) {
1374                                                df.removeLogicAccess();
1375                                        }
1376                                }
1377                        }
1378
1379                };
1380        }
1381
1382        public void setCopyStructureOnly(boolean copyStructureOnly) {
1383                this.copyStructureOnly = copyStructureOnly;
1384        }
1385
1386        public boolean isCopyStructureOnly() {
1387                return this.copyStructureOnly;
1388        }
1389
1390        private boolean isViEditSwp(String fileName) {
1391                return fileName.matches(TMP_MATCHES_EDIT) || fileName.matches(TMP_MATCHES_VI_SWP);
1392        }
1393
1394        protected static DiskFile copyAttrs(DiskFile source, boolean _copyStructureOnly, File _syncRoot) {
1395                source.setCopyStructureOnly(_copyStructureOnly);
1396                source.syncRoot = _syncRoot;
1397                return source;
1398        }
1399
1400        public File getParentFile() {
1401                return new File(getParent());
1402        }
1403
1404        protected boolean checkSyncPathAvailable(String path) {
1405                try {
1406                        return checkSyncPathAvailable(path, RemoteFile.SHOW_HIDDEN, RemoteFile.SHOW_LINK, RemoteFile.MAX_FILE_SZIE,
1407                                        SYNC_PATH_ALLOWED, SYNC_PATH_IGNORED);
1408                } catch (Exception e) {
1409                        LoggerFactory.getLogger(DiskFile.class).error(e.getMessage(), e);
1410                        return false;
1411                }
1412        }
1413
1414        protected static synchronized boolean addQueuePathMapping(String path) {
1415                Long addTimeMs = null;
1416                boolean allowed = checkQueuePathMapping();
1417
1418                if (!allowed)
1419                        return false;
1420
1421                addTimeMs = QUEUE_PATH_MAPPING.get(path);
1422
1423                if (addTimeMs == null) {
1424                        QUEUE_PATH_MAPPING.put(path, System.currentTimeMillis());
1425                        return true;
1426                }
1427
1428                return false;
1429        }
1430
1431        public static synchronized boolean checkQueuePathMapping() {
1432                Long addTimeMs = null;
1433                Map<String, Long> copyQueueMap = getQueuePathMapping();
1434                List<String> keyList = new ArrayList<String>(copyQueueMap.keySet());
1435                for (String key : keyList) {
1436                        addTimeMs = copyQueueMap.get(key);
1437                        if (addTimeMs != null) {
1438                                boolean timeout = (System.currentTimeMillis() - addTimeMs) >= LOGIC_TIMEOUT_MS;
1439
1440                                if (timeout) {
1441                                        QUEUE_PATH_MAPPING.remove(key);
1442                                }
1443                        }
1444                }
1445
1446                if (QUEUE_PATH_MAPPING.keySet().size() >= RemoteFile.MAX_QUEUE_SIZE)
1447                        return false;
1448
1449                return true;
1450        }
1451
1452        protected static synchronized void removeQueuePathMapping(String path) {
1453                QUEUE_PATH_MAPPING.remove(path);
1454        }
1455
1456        public static synchronized Map<String, Long> getQueuePathMapping() {
1457                return Collections.synchronizedMap(QUEUE_PATH_MAPPING);
1458        }
1459
1460        private static void debugLog(String point, RemoteFile rf, CacheArrayFilter filter) {
1461                debugLog(DiskFile.class, point, rf, filter);
1462        }
1463
1464}