Source code

001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import com.killcoding.log.LoggerFactory;
028import com.killcoding.log.Logger;
029import java.io.IOException;
030import java.util.List;
031import java.util.ArrayList;
032import java.nio.ByteBuffer;
033import java.io.ByteArrayInputStream;
034import java.sql.Timestamp;
035import com.killcoding.tool.CommonTools;
036import java.nio.file.Path;
037import java.nio.file.Files;
038import java.nio.file.Paths;
039import java.text.SimpleDateFormat;
040import java.time.ZonedDateTime;
041import java.time.ZoneId;
042import java.time.LocalDateTime;
043import java.time.format.DateTimeFormatter;
044import com.killcoding.cache.CacheArrayFilter;
045import java.nio.file.attribute.BasicFileAttributes;
046import java.nio.file.attribute.FileTime;
047import java.nio.file.attribute.FileAttribute;
048import java.nio.file.LinkOption;
049
050public abstract class BaseFile {
051    
052    protected static final String TMP_MATCHES_EDIT = "^.*\\..*~$";
053    protected static final String TMP_MATCHES_VI_SWP = "^.*\\..*\\.sw[xp]$";
054
055        protected static final String TMP_EXT_MODIFY_FILE = ".modify.tmp~";
056        protected static final String TMP_EXT_ACCESS_FILE = ".access.tmp~";
057        protected static final String TMP_EXT_CHECK_FILE = ".check.tmp~";
058        
059        protected static final String TMP_WRITING_SWP = "^.*\\.writing.tmp~$";
060        protected static final String TMP_WRITING_FOR_REPLACE = "\\.writing\\.tmp~";
061        protected static final String TMP_WRITING_FILE = ".writing.tmp~";
062        
063        public static String CHARSET = "UTF-8";
064
065        public abstract boolean isFile() throws IOException;
066        
067        public abstract boolean isDirectory() throws IOException;
068        
069        public abstract boolean isLink() throws IOException;
070
071        public abstract boolean delete() throws IOException;
072
073        public abstract boolean exists() throws IOException;
074
075        public abstract boolean mkdirs() throws IOException;
076        
077        public abstract boolean createLink(String target) throws IOException;
078        
079        public abstract String readLink() throws IOException;
080
081        public abstract boolean write(byte[] data, boolean append) throws IOException;
082
083        public abstract boolean write(byte[] data) throws IOException;
084
085        public abstract boolean write(String data, boolean append) throws IOException;
086
087        public abstract boolean write(String data) throws IOException;
088
089        public abstract byte[] readAllBytes() throws IOException;
090        
091        public abstract String readAllString() throws IOException;
092        
093        public abstract long size() throws IOException;
094        
095        public abstract boolean complete() throws IOException;
096        
097        public abstract Timestamp getModifyTime() throws IOException;
098        
099        public abstract void setModifyTime(Timestamp modifyTime) throws IOException;
100        
101        public abstract void copyTo(String toPath) throws IOException;
102        
103        public abstract void moveTo(String toPath) throws IOException;
104
105        public abstract boolean beforeDelete(boolean realDeleted);
106        
107        public abstract void afterDelete(boolean realDeleted);
108                
109        public abstract boolean beforeMkdirs();
110        
111        public abstract void afterMkdirs();
112        
113        public abstract boolean beforeCreateLink(String target);
114        
115        public abstract void afterCreateLink(String target);    
116        
117        public abstract boolean beforeWrite();
118        
119        public abstract void afterWrite();
120        
121        public static long CACHE_ARRAY_FILTER_TIMER = 10L;
122        
123        public static long LOGIC_TIMEOUT_MS = 300000L;
124        
125        public static long LOGIC_ACCESS_TIMEOUT_MS = 60000L;
126        
127        public static long LOGIC_CHECK_TIMEOUT_MS = 60000L;
128        
129        protected static boolean stopSync = true;
130        
131        protected File syncRoot = null;
132
133        protected File origin;
134        
135        public static Path TEMP_DIR = null;
136        
137        private static long debugLogTime = 0L;
138        
139        public static final List<String> SYNC_PATH_ALLOWED = new ArrayList<String>();
140        public static final List<String> SYNC_PATH_IGNORED = new ArrayList<String>();   
141        
142        static {
143            try{
144                SYNC_PATH_IGNORED.add(TMP_MATCHES_EDIT);
145                SYNC_PATH_IGNORED.add(TMP_MATCHES_VI_SWP);
146                
147                String tmpdir = System.getProperty("java.io.tmpdir");
148                Path tmpBfDir = Paths.get(String.format("%s/BaseFile",tmpdir));
149                if(!Files.exists(tmpBfDir)){
150                    Files.createDirectories(tmpBfDir);
151                }
152                Path pidDir = Paths.get(String.format("%s/%s/",tmpBfDir.toString(),ProcessHandle.current().pid()));
153                if(TEMP_DIR == null) TEMP_DIR = Files.createDirectories(pidDir);
154                
155            }catch(IOException e){
156                Logger.systemError(BaseFile.class,e.getMessage(),e);
157            }
158        }
159
160        public BaseFile(String path) {
161                super();
162                origin = new File(path);
163        }
164        
165        public BaseFile(File origin) {
166                super();
167                this.origin = origin;
168        }       
169        
170        public static void setTmpDir(String tmpDir) throws IOException {
171            TEMP_DIR = Files.createDirectories(Paths.get(tmpDir));
172        }
173
174        public String getPath() {
175        return parsePath(String.format("%s/%s",getParent(),origin.getName()));
176        }
177        
178        public File getOrigin(){
179            return origin;
180        }
181        
182        public String getParent() {
183                String parent = CommonTools.isBlank(origin.getParent()) ? "/" : origin.getParent();
184        return replacePath(parent);
185        }
186        
187        public List<String> getParents() {
188            List<String> parents = new ArrayList<String>();
189            String path = getPath();
190            do{
191                File file = new File(path);
192                path = file.getParent();
193                
194                if(path == null) break;
195                
196            parents.add(path);
197            }while(path != null);
198            
199            return parents;
200        }
201        
202        public void write(int partSize,byte[] allBytes) throws IOException {
203            try{
204            ByteBuffer bf = ByteBuffer.allocate(partSize);
205            ByteArrayInputStream bais = new ByteArrayInputStream(allBytes);
206            int partIndex = -1;
207            while(bais.read() != -1){
208                bf.flip();
209                        partIndex += 1;
210                        List<Byte> parts = new ArrayList<Byte>();
211                        while (bf.hasRemaining()) {
212                                parts.add(bf.get());
213                        }
214                        bf.clear();
215                        int partsSize = parts.size();
216                        byte[] part = new byte[partsSize];
217                        for (int i = 0; i < partsSize; i++) {
218                                part[i] = parts.get(i);
219                        }
220                        write(part,true);
221            }
222            }finally{
223                complete();
224            }
225        }       
226        
227    protected String toModifyTmpFilePath() throws IOException {
228        if(syncRoot == null) throw new IOException("The 'BaseFile.syncRoot' is null.");
229        
230        File syncFile = new File(getPath());
231        if(syncRoot != null){
232            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
233            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_MODIFY_FILE;
234            return replacedFilePath;
235        }
236        return null;  
237    }  
238    
239    protected String toAccessTmpFilePath() throws IOException {
240        if(syncRoot == null) throw new IOException("The 'BaseFile.syncRoot' is null.");
241        
242        File syncFile = new File(getPath());
243        if(syncRoot != null){
244            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
245            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_ACCESS_FILE;
246            return replacedFilePath;
247        }
248        return null;            
249    }  
250
251    protected String toCheckTmpFilePath() throws IOException {
252        if(syncRoot == null) throw new IOException("The 'BaseFile.syncRoot' is null.");
253        
254        File syncFile = new File(getPath());
255        if(syncRoot != null){
256            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
257            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_CHECK_FILE;
258            return replacedFilePath;
259        }
260        return null;            
261    }  
262    
263        public synchronized static void stopSync() {
264                LoggerFactory.getLogger(DiskFile.class).mark("Sending stop command.");
265        stopSync = true;
266                for (int i = 0; i < 29; i++) {
267                try{
268                    if(!isStopSync()) {
269                        stopSync();
270                        break;
271                    }
272                    Thread.sleep(1000L);
273                }catch(InterruptedException e){
274                    LoggerFactory.getLogger(BaseFile.class).error(e.getMessage(),e);
275                }
276                }
277                LoggerFactory.getLogger(DiskFile.class).mark("Sent stop command");
278        }
279        
280    public static boolean isStopSync(){
281            DiskFile.checkQueuePathMapping();
282            RemoteFile.checkQueuePathMapping();        
283        return stopSync && DiskFile.getQueuePathMapping().isEmpty() && RemoteFile.getQueuePathMapping().isEmpty();
284    }
285    
286    protected static boolean isDebug(RemoteFile remoteFile){
287        boolean debug = false;
288        if(remoteFile != null){
289            boolean fileDebug = remoteFile.getOrigin().getName().matches("^(\\.{0,1}DEBUG[\\.\\d\\w]*)$");
290            boolean logDeubg = remoteFile.getConfigProperties().getBoolean("LogCompletedStatus", false);
291                debug = fileDebug || logDeubg;
292        }
293        return debug || LoggerFactory.getLogger(BaseFile.class).isDebugEnabled();
294    }
295    
296    public static boolean isHidden(String path) throws Exception{
297        Path checkPath = Paths.get(path);
298
299        if(!Files.exists(checkPath)) return false;
300        
301        while(checkPath != null){
302
303            boolean hide = Files.isHidden(checkPath);
304            
305            if(hide) return true;
306            
307            checkPath = checkPath.getParent();
308            
309            
310        }
311                return false;
312    } 
313    
314    public static boolean checkSyncPathAvailable(String path, boolean showHidden,boolean showLink,long maxFileSize, List<String> syncPathAllowed,
315                        List<String> syncPathIgnored) {
316                            
317                if(maxFileSize <= 0) return false;
318                
319                try {
320                        boolean matchedAllowed = true;
321                        boolean matchedIgnored = false;
322                        Path fullPath = Paths.get(path);
323                        
324                        if(!showHidden){
325                            matchedAllowed = !isHidden(path);
326                        }
327                        
328                        if(matchedAllowed && !showLink && Files.exists(fullPath)){
329                            boolean isLink = Files.isSymbolicLink(fullPath);
330                            matchedAllowed = !isLink;
331                        }                       
332                
333                        if (matchedAllowed && syncPathAllowed != null) {
334                                for (String regex : syncPathAllowed) {
335                                        if (!CommonTools.isBlank(regex)) {
336                                                matchedAllowed = path.matches(regex);
337
338                                                if (matchedAllowed)
339                                                        break;
340                                        }
341                                }
342                        }
343                        if (matchedAllowed && syncPathIgnored != null) {
344                                for (String regex : syncPathIgnored) {
345                                        if (!CommonTools.isBlank(regex)) {
346                                                matchedIgnored = path.matches(regex);
347
348                                                if (matchedIgnored)
349                                                        break;
350                                        }
351                                }
352                        }
353
354                        boolean b = matchedAllowed && !matchedIgnored;
355                        if (b) {
356                                Path checkPath = Paths.get(path);
357                                if (Files.exists(checkPath)) {
358                                    
359                                    int parentLength = checkPath.getParent().toString().length();
360                                    int fileNamelength = checkPath.getFileName().toString().length();
361                                    
362                                    b = parentLength <= RemoteFile.MAX_FILE_PARENT_PATH_LENGTH && fileNamelength <= RemoteFile.MAX_FILE_NAME_LENGTH;
363                                    
364                                        if (b && Files.isRegularFile(checkPath)) {
365                                                long fileSize = Files.size(checkPath);
366                                                b = fileSize <= maxFileSize;
367                                        }
368                                }else{
369                                    b = false;
370                                }
371                        }
372                        
373                        if(b) {
374                        Path checkPath = Paths.get(path);
375                        if (Files.exists(checkPath)) {
376                            boolean isLink = Files.isSymbolicLink(checkPath);
377                            boolean isFile = checkPath.toFile().isFile();
378                            if(isFile && !isLink){
379                            if(RemoteFile.SYNC_CUTOFF_TIME == null){
380                                b = false;
381                            } else{
382                                long syncCutoffTimeMs = 0L;
383                                syncCutoffTimeMs = RemoteFile.SYNC_CUTOFF_TIME.getTime();
384                                        BasicFileAttributes attr = Files.readAttributes(checkPath, BasicFileAttributes.class);
385                                        FileTime fileTime = attr.lastModifiedTime();
386                                        
387                            b = fileTime.toMillis() >= syncCutoffTimeMs;
388                            }
389                            }
390                        }else{
391                            b = false;
392                        }
393                        }
394                        
395                        if (!b) {
396                                LoggerFactory.getLogger(BaseFile.class).debug("Unavailable: {}", path);
397                        }
398                        return b;
399                } catch (Exception e) {
400                        LoggerFactory.getLogger(BaseFile.class).error(e.getMessage(), e);
401                        return false;
402                }
403        }
404        
405        public boolean isDir() throws IOException {
406            return isDirectory();
407        }       
408        
409        public Timestamp timeZoneConver(String destTimeZone, String srcTimeZone, Timestamp srcDate) throws IOException {
410            try{
411            String pattern = "yyyy-MM-dd HH:mm:ss.SSS";
412                SimpleDateFormat fullFormat = new SimpleDateFormat(pattern);
413                DateTimeFormatter format = DateTimeFormatter.ofPattern(pattern);        
414                String srcDateStr = fullFormat.format(srcDate);
415                LocalDateTime ldt = LocalDateTime.parse(srcDateStr, DateTimeFormatter.ofPattern(pattern));
416                ZoneId srcZoneId = ZoneId.of(srcTimeZone);
417                ZonedDateTime srcDateTime = ldt.atZone(srcZoneId);
418                ZoneId destZoneId = ZoneId.of(destTimeZone);
419                ZonedDateTime destDateTime = srcDateTime.withZoneSameInstant(destZoneId);
420                return new Timestamp(fullFormat.parse(format.format(destDateTime)).getTime());
421            }catch(Exception e){
422               throw new IOException(e.getMessage(),e);
423            }
424        }       
425        
426        protected static String replacePath(String source){
427            return source.replaceAll("\\\\", "/");
428        }
429        
430        protected static String parsePath(String source){
431            return source.replaceAll("\\\\", "/").replaceAll("//","/");
432        }
433
434    public static boolean checkUsage(RemoteFile rf){
435        return memoryUsage() < RemoteFile.MAX_MEMORY_USAGE && rf.getDataSourceWriter().getUsage() < RemoteFile.MAX_CONNECT_USAGE && rf.getDataSourceReader().getUsage() < RemoteFile.MAX_CONNECT_USAGE;
436    }
437    
438
439    public static long memoryUsing(){
440        int mb = 1024 * 1024;
441                Runtime instance = Runtime.getRuntime();
442                long totalMemory = instance.totalMemory() / mb;
443                long freeMemory = instance.freeMemory() / mb;
444                return  (instance.totalMemory() - instance.freeMemory()) / mb;
445    }   
446
447    public static double memoryUsage(long using){
448        int mb = 1024 * 1024;
449                Runtime instance = Runtime.getRuntime();
450                long maxMemory = instance.maxMemory() / mb;
451                return (using * 1.0D / maxMemory * 1.0D);
452    }   
453    
454    public static double memoryUsage(){
455        return memoryUsage(memoryUsing());
456    }    
457    
458        protected synchronized static void debugLog(Class clazz,String point,RemoteFile rf,CacheArrayFilter filter){
459            long ctm = System.currentTimeMillis();
460            boolean canWrite = (ctm - debugLogTime) >= 60000L;
461            if(canWrite){
462                DiskFile.checkQueuePathMapping();
463                RemoteFile.checkQueuePathMapping();
464                debugLogTime = ctm;
465                long memoryUsing = memoryUsing();
466                String readerUsage = String.format("%.2f%%",rf.getDataSourceReader().getUsage()*100);
467                String writerUsage = String.format("%.2f%%",rf.getDataSourceWriter().getUsage()*100);
468                String memoryUsage = String.format("%.2f%%",memoryUsage(memoryUsing)*100);
469                LoggerFactory.getLogger(clazz).mark("************{}************",point);
470                LoggerFactory.getLogger(clazz).mark("[{}] DiskQueuePathMapping - {}",point,DiskFile.getQueuePathMapping().keySet().size());
471            LoggerFactory.getLogger(clazz).mark("[{}] RemoteQueuePathMapping - {}",point,RemoteFile.getQueuePathMapping().keySet().size());
472                LoggerFactory.getLogger(clazz).mark("[{}] CacheArrayPoolUsage - {}",point,filter.getCacheArray().getUsingPoolSize());
473                LoggerFactory.getLogger(clazz).mark("[{}] DataSourceReaderUsage - {}",point,readerUsage);       
474                LoggerFactory.getLogger(clazz).mark("[{}] DataSourceWriterUsage - {}",point,writerUsage);
475                LoggerFactory.getLogger(clazz).mark("[{}] MemoryUsage - {}mb ({})",point,memoryUsing,memoryUsage);
476            }
477        }       
478    
479}