Source code

001/*******************************************************************************
002The MIT License (MIT)
003
004Copyright (c) 2024 KILLCODING.COM
005
006Permission is hereby granted, free of charge, to any person obtaining a copy
007of this software and associated documentation files (the "Software"), to deal
008in the Software without restriction, including without limitation the rights
009to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
010copies of the Software, and to permit persons to whom the Software is
011furnished to do so, subject to the following conditions:
012
013The above copyright notice and this permission notice shall be included in
014all copies or substantial portions of the Software.
015
016THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
017IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
018FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
019AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
020LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
021OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
022THE SOFTWARE.
023*****************************************************************************/
024package com.killcoding.file;
025
026import java.io.File;
027import com.killcoding.log.LoggerFactory;
028import com.killcoding.log.Logger;
029import java.io.IOException;
030import java.util.List;
031import java.util.ArrayList;
032import java.nio.ByteBuffer;
033import java.io.ByteArrayInputStream;
034import java.sql.Timestamp;
035import com.killcoding.tool.CommonTools;
036import java.nio.file.Path;
037import java.nio.file.Files;
038import java.nio.file.Paths;
039import java.text.SimpleDateFormat;
040import java.time.ZonedDateTime;
041import java.time.ZoneId;
042import java.time.LocalDateTime;
043import java.time.format.DateTimeFormatter;
044import com.killcoding.cache.CacheArrayFilter;
045import java.nio.file.attribute.BasicFileAttributes;
046import java.nio.file.attribute.FileTime;
047import java.nio.file.attribute.FileAttribute;
048import java.nio.file.LinkOption;
049
050public abstract class BaseFile {
051    
052    protected static final String TMP_MATCHES_EDIT = "^.*\\..*~$";
053    protected static final String TMP_MATCHES_VI_SWP = "^.*\\..*\\.sw[xp]$";
054
055        protected static final String TMP_EXT_MODIFY_FILE = ".modify.tmp~";
056        protected static final String TMP_EXT_ACCESS_FILE = ".access.tmp~";
057        protected static final String TMP_EXT_CHECK_FILE = ".check.tmp~";
058        
059        protected static final String TMP_WRITING_SWP = "^.*\\.writing.tmp~$";
060        protected static final String TMP_WRITING_FOR_REPLACE = "\\.writing\\.tmp~";
061        protected static final String TMP_WRITING_FILE = ".writing.tmp~";
062        
063        public static String CHARSET = "UTF-8";
064
065        public abstract boolean isFile() throws IOException;
066        
067        public abstract boolean isDirectory() throws IOException;
068        
069        public abstract boolean isLink() throws IOException;
070
071        public abstract boolean delete() throws IOException;
072
073        public abstract boolean exists() throws IOException;
074
075        public abstract boolean mkdirs() throws IOException;
076        
077        public abstract boolean createLink(String target) throws IOException;
078        
079        public abstract String readLink() throws IOException;
080
081        public abstract boolean write(byte[] data, boolean append) throws IOException;
082
083        public abstract boolean write(byte[] data) throws IOException;
084
085        public abstract boolean write(String data, boolean append) throws IOException;
086
087        public abstract boolean write(String data) throws IOException;
088
089        public abstract byte[] readAllBytes() throws IOException;
090        
091        public abstract String readAllString() throws IOException;
092        
093        public abstract long size() throws IOException;
094        
095        public abstract boolean complete() throws IOException;
096        
097        public abstract Timestamp getModifyTime() throws IOException;
098        
099        public abstract void setModifyTime(Timestamp modifyTime) throws IOException;
100        
101        public abstract void copyTo(String toPath) throws IOException;
102        
103        public abstract void moveTo(String toPath) throws IOException;
104
105        public abstract boolean beforeDelete(boolean realDeleted);
106        
107        public abstract void afterDelete(boolean realDeleted);
108                
109        public abstract boolean beforeMkdirs();
110        
111        public abstract void afterMkdirs();
112        
113        public abstract boolean beforeCreateLink(String target);
114        
115        public abstract void afterCreateLink(String target);    
116        
117        public abstract boolean beforeWrite();
118        
119        public abstract void afterWrite();
120        
121        public static long CACHE_ARRAY_FILTER_TIMER = 10L;
122        
123        public static long LOGIC_TIMEOUT_MS = 300000L;
124        
125        public static long LOGIC_CHECK_TIMEOUT_MS = 60000L;
126        
127        protected static boolean stopSync = true;
128        
129        protected File syncRoot = null;
130
131        protected File origin;
132        
133        public static Path TEMP_DIR = null;
134        
135        private static long debugLogTime = 0L;
136        
137        public static final List<String> SYNC_PATH_ALLOWED = new ArrayList<String>();
138        public static final List<String> SYNC_PATH_IGNORED = new ArrayList<String>();   
139        
140        static {
141            try{
142                SYNC_PATH_IGNORED.add(TMP_MATCHES_EDIT);
143                SYNC_PATH_IGNORED.add(TMP_MATCHES_VI_SWP);
144                
145                String tmpdir = System.getProperty("java.io.tmpdir");
146                Path tmpBfDir = Paths.get(String.format("%s/BaseFile",tmpdir));
147                if(!Files.exists(tmpBfDir)){
148                    Files.createDirectories(tmpBfDir);
149                }
150                Path pidDir = Paths.get(String.format("%s/%s/",tmpBfDir.toString(),ProcessHandle.current().pid()));
151                if(TEMP_DIR == null) TEMP_DIR = Files.createDirectories(pidDir);
152                
153            }catch(IOException e){
154                Logger.systemError(BaseFile.class,e.getMessage(),e);
155            }
156        }
157
158        public BaseFile(String path) {
159                super();
160                origin = new File(path);
161        }
162        
163        public static void setTmpDir(String tmpDir) throws IOException {
164            TEMP_DIR = Files.createDirectories(Paths.get(tmpDir));
165        }
166
167        public String getPath() {
168        return parsePath(String.format("%s/%s",getParent(),origin.getName()));
169        }
170        
171        public File getOrigin(){
172            return origin;
173        }
174        
175        public String getParent() {
176                String parent = CommonTools.isBlank(origin.getParent()) ? "/" : origin.getParent();
177        return replacePath(parent);
178        }
179        
180        public List<String> getParents() {
181            List<String> parents = new ArrayList<String>();
182            String path = getPath();
183            do{
184                File file = new File(path);
185                path = file.getParent();
186                
187                if(path == null) break;
188                
189            parents.add(path);
190            }while(path != null);
191            
192            return parents;
193        }
194        
195        public void write(int partSize,byte[] allBytes) throws IOException {
196            try{
197            ByteBuffer bf = ByteBuffer.allocate(partSize);
198            ByteArrayInputStream bais = new ByteArrayInputStream(allBytes);
199            int partIndex = -1;
200            while(bais.read() != -1){
201                bf.flip();
202                        partIndex += 1;
203                        List<Byte> parts = new ArrayList<Byte>();
204                        while (bf.hasRemaining()) {
205                                parts.add(bf.get());
206                        }
207                        bf.clear();
208                        int partsSize = parts.size();
209                        byte[] part = new byte[partsSize];
210                        for (int i = 0; i < partsSize; i++) {
211                                part[i] = parts.get(i);
212                        }
213                        write(part,true);
214            }
215            }finally{
216                complete();
217            }
218        }       
219        
220    protected String toModifyTmpFilePath(){
221        if(syncRoot == null) return null;
222        
223        File syncFile = new File(getPath());
224        if(syncRoot != null){
225            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
226            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_MODIFY_FILE;
227            return replacedFilePath;
228        }
229        return null;  
230    }  
231    
232    protected String toAccessTmpFilePath(){
233        if(syncRoot == null) return null;
234        
235        File syncFile = new File(getPath());
236        if(syncRoot != null){
237            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
238            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_ACCESS_FILE;
239            return replacedFilePath;
240        }
241        return null;            
242    }  
243
244    protected String toCheckTmpFilePath(){
245        if(syncRoot == null) return null;
246        
247        File syncFile = new File(getPath());
248        if(syncRoot != null){
249            String syncRootParent = replacePath(syncRoot.getParent()) + "/";
250            String replacedFilePath = TEMP_DIR.toAbsolutePath() + "/." + replacePath(syncFile.getAbsolutePath()).replaceFirst(syncRootParent,"") + TMP_EXT_CHECK_FILE;
251            return replacedFilePath;
252        }
253        return null;            
254    }  
255        public synchronized static void stopSync() {
256                LoggerFactory.getLogger(DiskFile.class).mark("Sending stop command.");
257        stopSync = true;
258                for (int i = 0; i < 29; i++) {
259                try{
260                    if(!isStopSync()) {
261                        stopSync();
262                        break;
263                    }
264                    Thread.sleep(1000L);
265                }catch(InterruptedException e){
266                    LoggerFactory.getLogger(BaseFile.class).error(e.getMessage(),e);
267                }
268                }
269                LoggerFactory.getLogger(DiskFile.class).mark("Sent stop command");
270        }
271        
272    public static boolean isStopSync(){
273            DiskFile.checkQueuePathMapping();
274            RemoteFile.checkQueuePathMapping();        
275        return stopSync && DiskFile.getQueuePathMapping().isEmpty() && RemoteFile.getQueuePathMapping().isEmpty();
276    }
277    
278    protected static boolean isDebug(RemoteFile remoteFile){
279        boolean debug = false;
280        if(remoteFile != null){
281            boolean fileDebug = remoteFile.getOrigin().getName().matches("^(\\.{0,1}DEBUG[\\.\\d\\w]*)$");
282            boolean logDeubg = remoteFile.getConfigProperties().getBoolean("LogCompletedStatus", false);
283                debug = fileDebug || logDeubg;
284        }
285        return debug || LoggerFactory.getLogger(BaseFile.class).isDebugEnabled();
286    }
287    
288    public static boolean isHidden(String path) throws Exception{
289        Path checkPath = Paths.get(path);
290
291        if(!Files.exists(checkPath)) return false;
292        
293        while(checkPath != null){
294
295            boolean hide = Files.isHidden(checkPath);
296            
297            if(hide) return true;
298            
299            checkPath = checkPath.getParent();
300            
301            
302        }
303                return false;
304    } 
305    
306    public static boolean checkSyncPathAvailable(String path, boolean showHidden,boolean showLink,long maxFileSize, List<String> syncPathAllowed,
307                        List<String> syncPathIgnored) {
308                            
309                if(maxFileSize <= 0) return false;
310                
311                try {
312                        boolean matchedAllowed = true;
313                        boolean matchedIgnored = false;
314                        Path fullPath = Paths.get(path);
315                        
316                        if(!showHidden){
317                            matchedAllowed = !isHidden(path);
318                        }
319                        
320                        if(matchedAllowed && !showLink && Files.exists(fullPath)){
321                            boolean isLink = Files.isSymbolicLink(fullPath);
322                            matchedAllowed = !isLink;
323                        }                       
324                
325                        if (matchedAllowed && syncPathAllowed != null) {
326                                for (String regex : syncPathAllowed) {
327                                        if (!CommonTools.isBlank(regex)) {
328                                                matchedAllowed = path.matches(regex);
329
330                                                if (matchedAllowed)
331                                                        break;
332                                        }
333                                }
334                        }
335                        if (matchedAllowed && syncPathIgnored != null) {
336                                for (String regex : syncPathIgnored) {
337                                        if (!CommonTools.isBlank(regex)) {
338                                                matchedIgnored = path.matches(regex);
339
340                                                if (matchedIgnored)
341                                                        break;
342                                        }
343                                }
344                        }
345
346                        boolean b = matchedAllowed && !matchedIgnored;
347                        if (b) {
348                                Path checkPath = Paths.get(path);
349                                if (Files.exists(checkPath)) {
350                                    
351                                    int parentLength = checkPath.getParent().toString().length();
352                                    int fileNamelength = checkPath.getFileName().toString().length();
353                                    
354                                    b = parentLength <= RemoteFile.MAX_FILE_PARENT_PATH_LENGTH && fileNamelength <= RemoteFile.MAX_FILE_NAME_LENGTH;
355                                    
356                                        if (b && Files.isRegularFile(checkPath)) {
357                                                long fileSize = Files.size(checkPath);
358                                                b = fileSize <= maxFileSize;
359                                        }
360                                }else{
361                                    b = false;
362                                }
363                        }
364                        
365                        if(b) {
366                        Path checkPath = Paths.get(path);
367                        if (Files.exists(checkPath)) {
368                            boolean isLink = Files.isSymbolicLink(checkPath);
369                            boolean isFile = checkPath.toFile().isFile();
370                            if(isFile && !isLink){
371                            if(RemoteFile.SYNC_CUTOFF_TIME == null){
372                                b = false;
373                            } else{
374                                long syncCutoffTimeMs = 0L;
375                                syncCutoffTimeMs = RemoteFile.SYNC_CUTOFF_TIME.getTime();
376                                        BasicFileAttributes attr = Files.readAttributes(checkPath, BasicFileAttributes.class);
377                                        FileTime fileTime = attr.lastModifiedTime();
378                                        
379                            b = fileTime.toMillis() >= syncCutoffTimeMs;
380                            }
381                            }
382                        }else{
383                            b = false;
384                        }
385                        }
386                        
387                        if (!b) {
388                                LoggerFactory.getLogger(BaseFile.class).debug("Unavailable: {}", path);
389                        }
390                        return b;
391                } catch (Exception e) {
392                        LoggerFactory.getLogger(BaseFile.class).error(e.getMessage(), e);
393                        return false;
394                }
395        }
396        
397        public boolean isDir() throws IOException {
398            return isDirectory();
399        }       
400        
401        public Timestamp timeZoneConver(String destTimeZone, String srcTimeZone, Timestamp srcDate) throws IOException {
402            try{
403            String pattern = "yyyy-MM-dd HH:mm:ss.SSS";
404                SimpleDateFormat fullFormat = new SimpleDateFormat(pattern);
405                DateTimeFormatter format = DateTimeFormatter.ofPattern(pattern);        
406                String srcDateStr = fullFormat.format(srcDate);
407                LocalDateTime ldt = LocalDateTime.parse(srcDateStr, DateTimeFormatter.ofPattern(pattern));
408                ZoneId srcZoneId = ZoneId.of(srcTimeZone);
409                ZonedDateTime srcDateTime = ldt.atZone(srcZoneId);
410                ZoneId destZoneId = ZoneId.of(destTimeZone);
411                ZonedDateTime destDateTime = srcDateTime.withZoneSameInstant(destZoneId);
412                return new Timestamp(fullFormat.parse(format.format(destDateTime)).getTime());
413            }catch(Exception e){
414               throw new IOException(e.getMessage(),e);
415            }
416        }       
417        
418        protected static String replacePath(String source){
419            return source.replaceAll("\\\\", "/");
420        }
421        
422        protected static String parsePath(String source){
423            return source.replaceAll("\\\\", "/").replaceAll("//","/");
424        }
425
426    public static boolean checkUsage(RemoteFile rf){
427        return memoryUsage() < RemoteFile.MAX_MEMORY_USAGE && rf.getDataSourceWriter().getUsage() < RemoteFile.MAX_CONNECT_USAGE && rf.getDataSourceReader().getUsage() < RemoteFile.MAX_CONNECT_USAGE;
428    }
429    
430
431    public static long memoryUsing(){
432        int mb = 1024 * 1024;
433                Runtime instance = Runtime.getRuntime();
434                long totalMemory = instance.totalMemory() / mb;
435                long freeMemory = instance.freeMemory() / mb;
436                return  (instance.totalMemory() - instance.freeMemory()) / mb;
437    }   
438
439    public static double memoryUsage(long using){
440        int mb = 1024 * 1024;
441                Runtime instance = Runtime.getRuntime();
442                long maxMemory = instance.maxMemory() / mb;
443                return (using * 1.0D / maxMemory * 1.0D);
444    }   
445    
446    public static double memoryUsage(){
447        return memoryUsage(memoryUsing());
448    }    
449    
450        protected synchronized static void debugLog(Class clazz,String point,RemoteFile rf,CacheArrayFilter filter){
451            long ctm = System.currentTimeMillis();
452            boolean canWrite = (ctm - debugLogTime) >= 60000L;
453            if(canWrite){
454                DiskFile.checkQueuePathMapping();
455                RemoteFile.checkQueuePathMapping();
456                debugLogTime = ctm;
457                long memoryUsing = memoryUsing();
458                String readerUsage = String.format("%.2f%%",rf.getDataSourceReader().getUsage()*100);
459                String writerUsage = String.format("%.2f%%",rf.getDataSourceWriter().getUsage()*100);
460                String memoryUsage = String.format("%.2f%%",memoryUsage(memoryUsing)*100);
461                LoggerFactory.getLogger(clazz).mark("************{}************",point);
462                LoggerFactory.getLogger(clazz).mark("[{}] DiskQueuePathMapping - {}",point,DiskFile.getQueuePathMapping().keySet().size());
463            LoggerFactory.getLogger(clazz).mark("[{}] RemoteQueuePathMapping - {}",point,RemoteFile.getQueuePathMapping().keySet().size());
464                LoggerFactory.getLogger(clazz).mark("[{}] CacheArrayPoolUsage - {}",point,filter.getCacheArray().getUsingPoolSize());
465                LoggerFactory.getLogger(clazz).mark("[{}] DataSourceReaderUsage - {}",point,readerUsage);       
466                LoggerFactory.getLogger(clazz).mark("[{}] DataSourceWriterUsage - {}",point,writerUsage);
467                LoggerFactory.getLogger(clazz).mark("[{}] MemoryUsage - {}mb ({})",point,memoryUsing,memoryUsage);
468            }
469        }       
470    
471}