Source code

001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import com.killcoding.log.LoggerFactory;
028import com.killcoding.log.Logger;
029import java.io.IOException;
030import java.util.List;
031import java.util.ArrayList;
032import java.nio.ByteBuffer;
033import java.io.ByteArrayInputStream;
034import java.sql.Timestamp;
035import com.killcoding.tool.CommonTools;
036import java.nio.file.Path;
037import java.nio.file.Files;
038import java.nio.file.Paths;
039import java.text.SimpleDateFormat;
040import java.time.ZonedDateTime;
041import java.time.ZoneId;
042import java.time.LocalDateTime;
043import java.time.format.DateTimeFormatter;
044import com.killcoding.cache.CacheArrayFilter;
045import java.nio.file.attribute.BasicFileAttributes;
046import java.nio.file.attribute.FileTime;
047import java.nio.file.attribute.FileAttribute;
048import java.nio.file.LinkOption;
049
050public abstract class BaseFile {
051    
052    protected static final String TMP_MATCHES_EDIT = "^.*\\..*~$";
053    protected static final String TMP_MATCHES_VI_SWP = "^.*\\..*\\.sw[xp]$";
054
055        protected static final String TMP_EXT_MODIFY_FILE = ".modify.tmp~";
056        protected static final String TMP_EXT_ACCESS_FILE = ".access.tmp~";
057        protected static final String TMP_EXT_CHECK_FILE = ".check.tmp~";
058        
059        protected static final String TMP_WRITING_SWP = "^.*\\.writing.tmp~$";
060        protected static final String TMP_WRITING_FOR_REPLACE = "\\.writing\\.tmp~";
061        protected static final String TMP_WRITING_FILE = ".writing.tmp~";
062        
063        public static String CHARSET = "UTF-8";
064
065        public abstract boolean isFile() throws IOException;
066        
067        public abstract boolean isDirectory() throws IOException;
068        
069        public abstract boolean isLink() throws IOException;
070
071        public abstract boolean delete() throws IOException;
072
073        public abstract boolean exists() throws IOException;
074
075        public abstract boolean mkdirs() throws IOException;
076        
077        public abstract boolean createLink(String target) throws IOException;
078        
079        public abstract String readLink() throws IOException;
080
081        public abstract boolean write(byte[] data, boolean append) throws IOException;
082
083        public abstract boolean write(byte[] data) throws IOException;
084
085        public abstract boolean write(String data, boolean append) throws IOException;
086
087        public abstract boolean write(String data) throws IOException;
088
089        public abstract byte[] readAllBytes() throws IOException;
090        
091        public abstract String readAllString() throws IOException;
092        
093        public abstract long size() throws IOException;
094        
095        public abstract boolean complete() throws IOException;
096        
097        public abstract Timestamp getModifyTime() throws IOException;
098        
099        public abstract void setModifyTime(Timestamp modifyTime) throws IOException;
100        
101        public abstract void copyTo(String toPath) throws IOException;
102        
103        public abstract void moveTo(String toPath) throws IOException;
104
105        public abstract boolean beforeDelete(boolean realDeleted);
106        
107        public abstract void afterDelete(boolean realDeleted);
108                
109        public abstract boolean beforeMkdirs();
110        
111        public abstract void afterMkdirs();
112        
113        public abstract boolean beforeCreateLink(String target);
114        
115        public abstract void afterCreateLink(String target);    
116        
117        public abstract boolean beforeWrite();
118        
119        public abstract void afterWrite();
120        
121        public static long CACHE_ARRAY_FILTER_TIMER = 10L;
122        
123        public static long LOGIC_TIMEOUT_MS = 300000L;
124        
125        public static long LOGIC_ACCESS_TIMEOUT_MS = 60000L;
126        
127        public static long LOGIC_CHECK_TIMEOUT_MS = 60000L;
128        
129        protected static boolean stopSync = true;
130        
131        protected File syncRoot = null;
132
133        protected File origin;
134        
135        public static Path TEMP_DIR = null;
136        
137        private static long debugLogTime = 0L;
138        
139        public static final List<String> SYNC_PATH_ALLOWED = new ArrayList<String>();
140        public static final List<String> SYNC_PATH_IGNORED = new ArrayList<String>();   
141        
142        static {
143            try{
144                SYNC_PATH_IGNORED.add(TMP_MATCHES_EDIT);
145                SYNC_PATH_IGNORED.add(TMP_MATCHES_VI_SWP);
146                
147                String tmpdir = System.getProperty("java.io.tmpdir");
148                Path tmpBfDir = Paths.get(String.format("%s/BaseFile",tmpdir));
149                if(!Files.exists(tmpBfDir)){
150                    Files.createDirectories(tmpBfDir);
151                }
152                Path pidDir = Paths.get(String.format("%s/%s/",tmpBfDir.toString(),ProcessHandle.current().pid()));
153                if(TEMP_DIR == null) TEMP_DIR = Files.createDirectories(pidDir);
154                
155            }catch(IOException e){
156                Logger.systemError(BaseFile.class,e.getMessage(),e);
157            }
158        }
159
160        public BaseFile(String path) {
161                super();
162                origin = new File(path);
163        }
164        
165        public static void setTmpDir(String tmpDir) throws IOException {
166            TEMP_DIR = Files.createDirectories(Paths.get(tmpDir));
167        }
168
169        public String getPath() {
170        return parsePath(String.format("%s/%s",getParent(),origin.getName()));
171        }
172        
173        public File getOrigin(){
174            return origin;
175        }
176        
177        public String getParent() {
178                String parent = CommonTools.isBlank(origin.getParent()) ? "/" : origin.getParent();
179        return replacePath(parent);
180        }
181        
182        public List<String> getParents() {
183            List<String> parents = new ArrayList<String>();
184            String path = getPath();
185            do{
186                File file = new File(path);
187                path = file.getParent();
188                
189                if(path == null) break;
190                
191            parents.add(path);
192            }while(path != null);
193            
194            return parents;
195        }
196        
197        public void write(int partSize,byte[] allBytes) throws IOException {
198            try{
199            ByteBuffer bf = ByteBuffer.allocate(partSize);
200            ByteArrayInputStream bais = new ByteArrayInputStream(allBytes);
201            int partIndex = -1;
202            while(bais.read() != -1){
203                bf.flip();
204                        partIndex += 1;
205                        List<Byte> parts = new ArrayList<Byte>();
206                        while (bf.hasRemaining()) {
207                                parts.add(bf.get());
208                        }
209                        bf.clear();
210                        int partsSize = parts.size();
211                        byte[] part = new byte[partsSize];
212                        for (int i = 0; i < partsSize; i++) {
213                                part[i] = parts.get(i);
214                        }
215                        write(part,true);
216            }
217            }finally{
218                complete();
219            }
220        }       
221        
222    protected String toModifyTmpFilePath() throws IOException {
223        if(syncRoot == null) throw new IOException("The 'BaseFile.syncRoot' is null.");
224        
225        File syncFile = new File(getPath());
226        if(syncRoot != null){
227            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
228            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_MODIFY_FILE;
229            return replacedFilePath;
230        }
231        return null;  
232    }  
233    
234    protected String toAccessTmpFilePath() throws IOException {
235        if(syncRoot == null) throw new IOException("The 'BaseFile.syncRoot' is null.");
236        
237        File syncFile = new File(getPath());
238        if(syncRoot != null){
239            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
240            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_ACCESS_FILE;
241            return replacedFilePath;
242        }
243        return null;            
244    }  
245
246    protected String toCheckTmpFilePath() throws IOException {
247        if(syncRoot == null) throw new IOException("The 'BaseFile.syncRoot' is null.");
248        
249        File syncFile = new File(getPath());
250        if(syncRoot != null){
251            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
252            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_CHECK_FILE;
253            return replacedFilePath;
254        }
255        return null;            
256    }  
257    
258        public synchronized static void stopSync() {
259                LoggerFactory.getLogger(DiskFile.class).mark("Sending stop command.");
260        stopSync = true;
261                for (int i = 0; i < 29; i++) {
262                try{
263                    if(!isStopSync()) {
264                        stopSync();
265                        break;
266                    }
267                    Thread.sleep(1000L);
268                }catch(InterruptedException e){
269                    LoggerFactory.getLogger(BaseFile.class).error(e.getMessage(),e);
270                }
271                }
272                LoggerFactory.getLogger(DiskFile.class).mark("Sent stop command");
273        }
274        
275    public static boolean isStopSync(){
276            DiskFile.checkQueuePathMapping();
277            RemoteFile.checkQueuePathMapping();        
278        return stopSync && DiskFile.getQueuePathMapping().isEmpty() && RemoteFile.getQueuePathMapping().isEmpty();
279    }
280    
281    protected static boolean isDebug(RemoteFile remoteFile){
282        boolean debug = false;
283        if(remoteFile != null){
284            boolean fileDebug = remoteFile.getOrigin().getName().matches("^(\\.{0,1}DEBUG[\\.\\d\\w]*)$");
285            boolean logDeubg = remoteFile.getConfigProperties().getBoolean("LogCompletedStatus", false);
286                debug = fileDebug || logDeubg;
287        }
288        return debug || LoggerFactory.getLogger(BaseFile.class).isDebugEnabled();
289    }
290    
291    public static boolean isHidden(String path) throws Exception{
292        Path checkPath = Paths.get(path);
293
294        if(!Files.exists(checkPath)) return false;
295        
296        while(checkPath != null){
297
298            boolean hide = Files.isHidden(checkPath);
299            
300            if(hide) return true;
301            
302            checkPath = checkPath.getParent();
303            
304            
305        }
306                return false;
307    } 
308    
309    public static boolean checkSyncPathAvailable(String path, boolean showHidden,boolean showLink,long maxFileSize, List<String> syncPathAllowed,
310                        List<String> syncPathIgnored) {
311                            
312                if(maxFileSize <= 0) return false;
313                
314                try {
315                        boolean matchedAllowed = true;
316                        boolean matchedIgnored = false;
317                        Path fullPath = Paths.get(path);
318                        
319                        if(!showHidden){
320                            matchedAllowed = !isHidden(path);
321                        }
322                        
323                        if(matchedAllowed && !showLink && Files.exists(fullPath)){
324                            boolean isLink = Files.isSymbolicLink(fullPath);
325                            matchedAllowed = !isLink;
326                        }                       
327                
328                        if (matchedAllowed && syncPathAllowed != null) {
329                                for (String regex : syncPathAllowed) {
330                                        if (!CommonTools.isBlank(regex)) {
331                                                matchedAllowed = path.matches(regex);
332
333                                                if (matchedAllowed)
334                                                        break;
335                                        }
336                                }
337                        }
338                        if (matchedAllowed && syncPathIgnored != null) {
339                                for (String regex : syncPathIgnored) {
340                                        if (!CommonTools.isBlank(regex)) {
341                                                matchedIgnored = path.matches(regex);
342
343                                                if (matchedIgnored)
344                                                        break;
345                                        }
346                                }
347                        }
348
349                        boolean b = matchedAllowed && !matchedIgnored;
350                        if (b) {
351                                Path checkPath = Paths.get(path);
352                                if (Files.exists(checkPath)) {
353                                    
354                                    int parentLength = checkPath.getParent().toString().length();
355                                    int fileNamelength = checkPath.getFileName().toString().length();
356                                    
357                                    b = parentLength <= RemoteFile.MAX_FILE_PARENT_PATH_LENGTH && fileNamelength <= RemoteFile.MAX_FILE_NAME_LENGTH;
358                                    
359                                        if (b && Files.isRegularFile(checkPath)) {
360                                                long fileSize = Files.size(checkPath);
361                                                b = fileSize <= maxFileSize;
362                                        }
363                                }else{
364                                    b = false;
365                                }
366                        }
367                        
368                        if(b) {
369                        Path checkPath = Paths.get(path);
370                        if (Files.exists(checkPath)) {
371                            boolean isLink = Files.isSymbolicLink(checkPath);
372                            boolean isFile = checkPath.toFile().isFile();
373                            if(isFile && !isLink){
374                            if(RemoteFile.SYNC_CUTOFF_TIME == null){
375                                b = false;
376                            } else{
377                                long syncCutoffTimeMs = 0L;
378                                syncCutoffTimeMs = RemoteFile.SYNC_CUTOFF_TIME.getTime();
379                                        BasicFileAttributes attr = Files.readAttributes(checkPath, BasicFileAttributes.class);
380                                        FileTime fileTime = attr.lastModifiedTime();
381                                        
382                            b = fileTime.toMillis() >= syncCutoffTimeMs;
383                            }
384                            }
385                        }else{
386                            b = false;
387                        }
388                        }
389                        
390                        if (!b) {
391                                LoggerFactory.getLogger(BaseFile.class).debug("Unavailable: {}", path);
392                        }
393                        return b;
394                } catch (Exception e) {
395                        LoggerFactory.getLogger(BaseFile.class).error(e.getMessage(), e);
396                        return false;
397                }
398        }
399        
400        public boolean isDir() throws IOException {
401            return isDirectory();
402        }       
403        
404        public Timestamp timeZoneConver(String destTimeZone, String srcTimeZone, Timestamp srcDate) throws IOException {
405            try{
406            String pattern = "yyyy-MM-dd HH:mm:ss.SSS";
407                SimpleDateFormat fullFormat = new SimpleDateFormat(pattern);
408                DateTimeFormatter format = DateTimeFormatter.ofPattern(pattern);        
409                String srcDateStr = fullFormat.format(srcDate);
410                LocalDateTime ldt = LocalDateTime.parse(srcDateStr, DateTimeFormatter.ofPattern(pattern));
411                ZoneId srcZoneId = ZoneId.of(srcTimeZone);
412                ZonedDateTime srcDateTime = ldt.atZone(srcZoneId);
413                ZoneId destZoneId = ZoneId.of(destTimeZone);
414                ZonedDateTime destDateTime = srcDateTime.withZoneSameInstant(destZoneId);
415                return new Timestamp(fullFormat.parse(format.format(destDateTime)).getTime());
416            }catch(Exception e){
417               throw new IOException(e.getMessage(),e);
418            }
419        }       
420        
421        protected static String replacePath(String source){
422            return source.replaceAll("\\\\", "/");
423        }
424        
425        protected static String parsePath(String source){
426            return source.replaceAll("\\\\", "/").replaceAll("//","/");
427        }
428
429    public static boolean checkUsage(RemoteFile rf){
430        return memoryUsage() < RemoteFile.MAX_MEMORY_USAGE && rf.getDataSourceWriter().getUsage() < RemoteFile.MAX_CONNECT_USAGE && rf.getDataSourceReader().getUsage() < RemoteFile.MAX_CONNECT_USAGE;
431    }
432    
433
434    public static long memoryUsing(){
435        int mb = 1024 * 1024;
436                Runtime instance = Runtime.getRuntime();
437                long totalMemory = instance.totalMemory() / mb;
438                long freeMemory = instance.freeMemory() / mb;
439                return  (instance.totalMemory() - instance.freeMemory()) / mb;
440    }   
441
442    public static double memoryUsage(long using){
443        int mb = 1024 * 1024;
444                Runtime instance = Runtime.getRuntime();
445                long maxMemory = instance.maxMemory() / mb;
446                return (using * 1.0D / maxMemory * 1.0D);
447    }   
448    
449    public static double memoryUsage(){
450        return memoryUsage(memoryUsing());
451    }    
452    
453        protected synchronized static void debugLog(Class clazz,String point,RemoteFile rf,CacheArrayFilter filter){
454            long ctm = System.currentTimeMillis();
455            boolean canWrite = (ctm - debugLogTime) >= 60000L;
456            if(canWrite){
457                DiskFile.checkQueuePathMapping();
458                RemoteFile.checkQueuePathMapping();
459                debugLogTime = ctm;
460                long memoryUsing = memoryUsing();
461                String readerUsage = String.format("%.2f%%",rf.getDataSourceReader().getUsage()*100);
462                String writerUsage = String.format("%.2f%%",rf.getDataSourceWriter().getUsage()*100);
463                String memoryUsage = String.format("%.2f%%",memoryUsage(memoryUsing)*100);
464                LoggerFactory.getLogger(clazz).mark("************{}************",point);
465                LoggerFactory.getLogger(clazz).mark("[{}] DiskQueuePathMapping - {}",point,DiskFile.getQueuePathMapping().keySet().size());
466            LoggerFactory.getLogger(clazz).mark("[{}] RemoteQueuePathMapping - {}",point,RemoteFile.getQueuePathMapping().keySet().size());
467                LoggerFactory.getLogger(clazz).mark("[{}] CacheArrayPoolUsage - {}",point,filter.getCacheArray().getUsingPoolSize());
468                LoggerFactory.getLogger(clazz).mark("[{}] DataSourceReaderUsage - {}",point,readerUsage);       
469                LoggerFactory.getLogger(clazz).mark("[{}] DataSourceWriterUsage - {}",point,writerUsage);
470                LoggerFactory.getLogger(clazz).mark("[{}] MemoryUsage - {}mb ({})",point,memoryUsing,memoryUsage);
471            }
472        }       
473    
474}