Reimplemented much of the kqueue-based file watcher so that watching subdirectories...
authorCody Russell <cody@jhu.edu>
Sun, 8 Jun 2014 21:46:18 +0000 (16:46 -0500)
committerAlexis Christoforides <alexis@thenull.net>
Wed, 16 Jul 2014 21:27:20 +0000 (17:27 -0400)
https://bugzilla.xamarin.com/show_bug.cgi?id=16259

mcs/class/System/System.IO/KeventWatcher.cs

index e673f99739bcaecc9623faa6a277b7c5ca60ef44..5dd52d42c020c0d42e06e7745c86e95b6afbae18 100644 (file)
@@ -3,6 +3,7 @@
 //
 // Authors:
 //     Geoff Norton (gnorton@customerdna.com)
+//  Cody Russell (cody@xamarin.com)
 //
 // (c) 2004 Geoff Norton
 // Copyright 2014 Xamarin Inc
@@ -28,7 +29,9 @@
 //
 
 using System;
+using System.IO;
 using System.Collections;
+using System.Collections.Generic;
 using System.ComponentModel;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -36,100 +39,100 @@ using System.Text;
 using System.Threading;
 
 namespace System.IO {
-
-        [Flags]
-        enum EventFlags : ushort {
-                Add         = 0x0001,
-                Delete      = 0x0002,
-                Enable      = 0x0004,
-                Disable     = 0x0008,
-                OneShot     = 0x0010,
-                Clear       = 0x0020,
-                Receipt     = 0x0040,
-                Dispatch    = 0x0080,
-
-                Flag0       = 0x1000,
-                Flag1       = 0x2000,
-                SystemFlags = unchecked (0xf000),
-                        
-                // Return values.
-                EOF         = 0x8000,
-                Error       = 0x4000,
-        }
+       [Flags]
+       enum EventFlags : ushort {
+               Add         = 0x0001,
+               Delete      = 0x0002,
+               Enable      = 0x0004,
+               Disable     = 0x0008,
+               OneShot     = 0x0010,
+               Clear       = 0x0020,
+               Receipt     = 0x0040,
+               Dispatch    = 0x0080,
+
+               Flag0       = 0x1000,
+               Flag1       = 0x2000,
+               SystemFlags = unchecked (0xf000),
+
+               // Return values.
+               EOF         = 0x8000,
+               Error       = 0x4000,
+       }
         
-        enum EventFilter : short {
-                Read = -1,
-                Write = -2,
-                Aio = -3,
-                Vnode = -4,
-                Proc = -5,
-                Signal = -6,
-                Timer = -7,
-                MachPort = -8,
-                FS = -9,
-                User = -10,
-                VM = -11
-        }
+       enum EventFilter : short {
+               Read     = -1,
+               Write    = -2,
+               Aio      = -3,
+               Vnode    = -4,
+               Proc     = -5,
+               Signal   = -6,
+               Timer    = -7,
+               MachPort = -8,
+               FS       = -9,
+               User     = -10,
+               VM       = -11
+       }
 
        enum FilterFlags : uint {
-                ReadPoll          = EventFlags.Flag0,
-                ReadOutOfBand     = EventFlags.Flag1,
-                ReadLowWaterMark  = 0x00000001,
-
-                WriteLowWaterMark = ReadLowWaterMark,
-
-                NoteTrigger       = 0x01000000,
-                NoteFFNop         = 0x00000000,
-                NoteFFAnd         = 0x40000000,
-                NoteFFOr          = 0x80000000,
-                NoteFFCopy        = 0xc0000000,
-                NoteFFCtrlMask    = 0xc0000000,
-                NoteFFlagsMask    = 0x00ffffff,
-                                  
-                VNodeDelete       = 0x00000001,
-                VNodeWrite        = 0x00000002,
-                VNodeExtend       = 0x00000004,
-                VNodeAttrib       = 0x00000008,
-                VNodeLink         = 0x00000010,
-                VNodeRename       = 0x00000020,
-                VNodeRevoke       = 0x00000040,
-                VNodeNone         = 0x00000080,
-                                  
-                ProcExit          = 0x80000000,
-                ProcFork          = 0x40000000,
-                ProcExec          = 0x20000000,
-                ProcReap          = 0x10000000,
-                ProcSignal        = 0x08000000,
-                ProcExitStatus    = 0x04000000,
-                ProcResourceEnd   = 0x02000000,
-
-                // iOS only
-                ProcAppactive     = 0x00800000,
-                ProcAppBackground = 0x00400000,
-                ProcAppNonUI      = 0x00200000,
-                ProcAppInactive   = 0x00100000,
-                ProcAppAllStates  = 0x00f00000,
-
-                // Masks
-                ProcPDataMask     = 0x000fffff,
-                ProcControlMask   = 0xfff00000,
-
-                VMPressure        = 0x80000000,
-                VMPressureTerminate = 0x40000000,
-                VMPressureSuddenTerminate = 0x20000000,
-                VMError           = 0x10000000,
-                TimerSeconds      =    0x00000001,
-                TimerMicroSeconds =   0x00000002,
-                TimerNanoSeconds  =   0x00000004,
-                TimerAbsolute     =   0x00000008,
-        }
-                       
+               ReadPoll          = EventFlags.Flag0,
+               ReadOutOfBand     = EventFlags.Flag1,
+               ReadLowWaterMark  = 0x00000001,
+
+               WriteLowWaterMark = ReadLowWaterMark,
+
+               NoteTrigger       = 0x01000000,
+               NoteFFNop         = 0x00000000,
+               NoteFFAnd         = 0x40000000,
+               NoteFFOr          = 0x80000000,
+               NoteFFCopy        = 0xc0000000,
+               NoteFFCtrlMask    = 0xc0000000,
+               NoteFFlagsMask    = 0x00ffffff,
+
+               VNodeDelete       = 0x00000001,
+               VNodeWrite        = 0x00000002,
+               VNodeExtend       = 0x00000004,
+               VNodeAttrib       = 0x00000008,
+               VNodeLink         = 0x00000010,
+               VNodeRename       = 0x00000020,
+               VNodeRevoke       = 0x00000040,
+               VNodeNone         = 0x00000080,
+
+               ProcExit          = 0x80000000,
+               ProcFork          = 0x40000000,
+               ProcExec          = 0x20000000,
+               ProcReap          = 0x10000000,
+               ProcSignal        = 0x08000000,
+               ProcExitStatus    = 0x04000000,
+               ProcResourceEnd   = 0x02000000,
+
+               // iOS only
+               ProcAppactive     = 0x00800000,
+               ProcAppBackground = 0x00400000,
+               ProcAppNonUI      = 0x00200000,
+               ProcAppInactive   = 0x00100000,
+               ProcAppAllStates  = 0x00f00000,
+
+               // Masks
+               ProcPDataMask     = 0x000fffff,
+               ProcControlMask   = 0xfff00000,
+
+               VMPressure                = 0x80000000,
+               VMPressureTerminate       = 0x40000000,
+               VMPressureSuddenTerminate = 0x20000000,
+               VMError                   = 0x10000000,
+               TimerSeconds              = 0x00000001,
+               TimerMicroSeconds         = 0x00000002,
+               TimerNanoSeconds          = 0x00000004,
+               TimerAbsolute             = 0x00000008,
+       }
+
+       [StructLayout(LayoutKind.Sequential)]
        struct kevent : IDisposable {
                public int ident;
                public EventFilter filter;
                public EventFlags flags;
                public FilterFlags fflags;
-               public int data;
+               public IntPtr data;
                public IntPtr udata;
 
                public void Dispose ()
@@ -157,24 +160,30 @@ namespace System.IO {
        }
 
        class KeventData {
-                public FileSystemWatcher FSW;
-                public string Directory;
-                public string FileMask;
-                public bool IncludeSubdirs;
-                public bool Enabled;
+               public FileSystemWatcher FSW;
+               public string Path;
+               public string FileMask;
+               public bool IncludeSubdirs;
+               public bool Enabled;
                public Hashtable DirEntries;
                public kevent ev;
-        }
+               public int fd;
+               public bool IsDirectory;
+       }
 
        class KeventWatcher : IFileWatcher
        {
                static bool failed;
                static KeventWatcher instance;
-               static Hashtable watches;
-               static Hashtable requests;
+               static Hashtable watches;  // <FileSystemWatcher, KeventData>
                static Thread thread;
                static int conn;
                static bool stop;
+
+               readonly Dictionary<string, KeventData> filenamesDict = new Dictionary<string, KeventData> ();
+               readonly Dictionary<int, KeventData> fdsDict = new Dictionary<int, KeventData> ();
+               readonly List<int> removeQueue = new List<int> ();
+               readonly List<int> rescanQueue = new List<int> ();
                
                private KeventWatcher ()
                {
@@ -194,7 +203,6 @@ namespace System.IO {
                        }
 
                        watches = Hashtable.Synchronized (new Hashtable ());
-                       requests = Hashtable.Synchronized (new Hashtable ());
                        conn = kqueue();
                        if (conn == -1) {
                                failed = true;
@@ -210,6 +218,7 @@ namespace System.IO {
                public void StartDispatching (FileSystemWatcher fsw)
                {
                        KeventData data;
+
                        lock (this) {
                                if (thread == null) {
                                        thread = new Thread (new ThreadStart (Monitor));
@@ -223,58 +232,108 @@ namespace System.IO {
                        if (data == null) {
                                data = new KeventData ();
                                data.FSW = fsw;
-                               data.Directory = fsw.FullPath;
+                               data.Path = fsw.FullPath;
                                data.FileMask = fsw.MangledFilter;
                                data.IncludeSubdirs = fsw.IncludeSubdirectories;
 
                                data.Enabled = true;
                                lock (this) {
-                                       StartMonitoringDirectory (data);
+                                       Scan (data);
                                        watches [fsw] = data;
                                        stop = false;
                                }
                        }
                }
 
-               static void StartMonitoringDirectory (KeventData data)
+               bool Add (KeventData data, bool postEvents = false)
                {
-                       DirectoryInfo dir = new DirectoryInfo (data.Directory);
-                       if(data.DirEntries == null) {
-                               data.DirEntries = new Hashtable();
-                               foreach (FileSystemInfo fsi in dir.GetFileSystemInfos() ) 
-                                       data.DirEntries.Add(fsi.FullName, new KeventFileData(fsi, fsi.LastAccessTime, fsi.LastWriteTime));
+                       var path = data.Path;
+
+                       if (filenamesDict.ContainsKey (path) || fdsDict.ContainsKey (data.fd) ) {
+                               return false;
                        }
 
-                       int fd = open(data.Directory, 0, 0);
-                       kevent ev = new kevent();
-                       ev.udata = IntPtr.Zero;
-                       timespec nullts = new timespec();
-                       nullts.tv_sec = 0;
-                       nullts.tv_usec = 0;
-                       if (fd > 0) {
-                               ev.ident = fd;
-                               ev.filter = EventFilter.Vnode;
-                               ev.flags = EventFlags.Add | EventFlags.Enable | EventFlags.OneShot;
-                               ev.fflags = // 20 | 2 | 1 | 8;
-                                       FilterFlags.VNodeDelete |
-                                       FilterFlags.VNodeWrite |
-                                       FilterFlags.VNodeAttrib |
-                                       // The following two values are the equivalent of the original value "20", but we suspect the original author meant
-                                       // 0x20, we will review later with some test cases
-                                       FilterFlags.VNodeLink |
-                                       FilterFlags.VNodeExtend;
-                               ev.data = 0;
-                               ev.udata = Marshal.StringToHGlobalAuto (data.Directory);
-                               kevent outev = new kevent();
-                               outev.udata = IntPtr.Zero;
-                               kevent (conn, ref ev, 1, ref outev, 0, ref nullts);
-                               data.ev = ev;
-                               requests [fd] = data;
+                       var fd = open (path, 0x8000 /* O_EVTONLY */, 0);
+
+                       if (fd != -1) {
+                               data.fd = fd;
+                               filenamesDict.Add (path, data);
+                               fdsDict.Add (fd, data);
+
+                               var attrs = File.GetAttributes (data.Path);
+                               data.IsDirectory = ((attrs & FileAttributes.Directory) == FileAttributes.Directory);
+
+                               if (postEvents)
+                                       PostEvent (path, data.FSW, FileAction.Added, path);
+
+                               return true;
+                       } else {
+                               return false;
                        }
-                       
-                       if (!data.IncludeSubdirs)
+               }
+
+               void Remove (int fd)
+               {
+                       if (!fdsDict.ContainsKey (fd))
                                return;
 
+                       var data = fdsDict [fd];
+                       fdsDict.Remove (fd);
+                       filenamesDict.Remove (data.Path);
+                       removeQueue.Remove (fd);
+
+                       close (fd);
+               }
+
+               void Remove (string path)
+               {
+                       var data = filenamesDict [path];
+
+                       filenamesDict.Remove (path);
+                       fdsDict.Remove (data.fd);
+                       close (data.fd);
+               }
+
+               bool Scan (KeventData data, bool postEvents = false)
+               {
+                       var path = data.Path;
+
+                       Add (data);
+                       if (!data.IncludeSubdirs) {
+                               return true;
+                       }
+
+                       if (data.IsDirectory && !Directory.Exists (path))
+                               return false;
+
+                       var attrs = File.GetAttributes (path);
+                       if ((attrs & FileAttributes.Directory) == FileAttributes.Directory) {
+                               var dirs_to_process = new List<string> ();
+                               dirs_to_process.Add (path);
+
+                               while (dirs_to_process.Count > 0) {
+                                       var tmp_path = dirs_to_process [0];
+                                       dirs_to_process.RemoveAt (0);
+                                       var dirinfo = new DirectoryInfo (tmp_path);
+                                       foreach (var fsi in dirinfo.GetFileSystemInfos ()) {
+                                               var newdata = new KeventData {
+                                                       Path = fsi.FullName,
+                                                       FileMask = data.FileMask,
+                                                       FSW = data.FSW,
+                                                       IncludeSubdirs = data.IncludeSubdirs
+                                               };
+
+                                               if (!Add (newdata, postEvents))
+                                                       continue;
+
+                                               var childAttrs = File.GetAttributes (fsi.FullName);
+                                               if ((childAttrs & FileAttributes.Directory) == FileAttributes.Directory)
+                                                       dirs_to_process.Add (fsi.FullName);
+                                       }
+                               }
+                       }
+
+                       return true;
                }
 
                public void StopDispatching (FileSystemWatcher fsw)
@@ -303,140 +362,107 @@ namespace System.IO {
 
                void Monitor ()
                {
-               
+                       bool firstRun = true;
+
                        while (!stop) {
-                               kevent ev = new kevent();
-                               ev.udata = IntPtr.Zero;
-                               kevent nullev = new kevent();
-                               nullev.udata = IntPtr.Zero;
-                               timespec ts = new timespec();
-                               ts.tv_sec = 0;
-                               ts.tv_usec = 0;
-                               int haveEvents;
-                               lock (this) {
-                                       haveEvents = kevent (conn, ref nullev, 0, ref ev, 1, ref ts);
+                               removeQueue.ForEach (Remove);
+
+                               rescanQueue.ForEach (
+                                       fd => {
+                                               var data = fdsDict[fd];
+                                               Scan (data, !firstRun);
+
+                                               rescanQueue.Remove (fd);
+                                       }
+                               );
+
+                               foreach (KeventData data in watches.Values) {
+                                       Scan (data);
                                }
 
-                               if (haveEvents > 0) {
-                                       // Restart monitoring
-                                       KeventData data = (KeventData) requests [ev.ident];
-                                       StopMonitoringDirectory (data);
-                                       StartMonitoringDirectory (data);
-                                       ProcessEvent (ev);
-                               } else {
-                                       System.Threading.Thread.Sleep (500);
+                               var changes = new List<kevent> ();
+                               var outEvents = new List<kevent> ();
+
+                               foreach (KeyValuePair<int, KeventData> kv in fdsDict) {
+                                       var change = new kevent {
+                                               ident = kv.Key,
+                                               filter = EventFilter.Vnode,
+                                               flags = EventFlags.Add | EventFlags.Enable | EventFlags.Clear,
+                                               fflags = FilterFlags.VNodeDelete | FilterFlags.VNodeExtend | FilterFlags.VNodeRename | FilterFlags.VNodeAttrib | FilterFlags.VNodeLink | FilterFlags.VNodeRevoke | FilterFlags.VNodeWrite,
+                                               data = IntPtr.Zero,
+                                               udata = IntPtr.Zero
+                                       };
+
+                                       changes.Add (change);
+                                       outEvents.Add (new kevent());
                                }
-                       }
 
-                       lock (this) {
-                               thread = null;
-                               stop = false;
-                       }
-               }
+                               if (changes.Count > 0) {
+                                       int numEvents = 0;
+                                       var out_array = outEvents.ToArray ();
 
-               void ProcessEvent (kevent ev)
-               {
-                       lock (this) {
-                               KeventData data = (KeventData) requests [ev.ident];
-                               if (!data.Enabled)
-                                       return;
+                                       lock (this) {
+                                               kevent[] changes_array = changes.ToArray ();
+                                               numEvents = kevent (conn, changes_array, changes_array.Length, out_array, out_array.Length, IntPtr.Zero);
+                                       }
 
-                               FileSystemWatcher fsw;
-                               string filename = "";
-
-                               fsw = data.FSW;
-                               FileAction fa = 0;
-                               DirectoryInfo dir = new DirectoryInfo (data.Directory);
-                               FileSystemInfo changedFsi = null;
-
-                               try {
-                                       foreach (FileSystemInfo fsi in dir.GetFileSystemInfos() )
-                                               if (data.DirEntries.ContainsKey (fsi.FullName) && (fsi is FileInfo)) {
-                                                       KeventFileData entry = (KeventFileData) data.DirEntries [fsi.FullName];
-                                                       if (entry.LastWriteTime != fsi.LastWriteTime) {
-                                                               filename = fsi.Name;
-                                                               fa = FileAction.Modified;
-                                                               data.DirEntries [fsi.FullName] = new KeventFileData(fsi, fsi.LastAccessTime, fsi.LastWriteTime);
-                                                               if (fsw.IncludeSubdirectories && fsi is DirectoryInfo) {
-                                                                       data.Directory = filename;
-                                                                       requests [ev.ident] = data;
-                                                                       ProcessEvent(ev);
-                                                               }
-                                                               changedFsi = fsi;
-                                                               PostEvent(filename, fsw, fa, changedFsi);
-                                                       }
-                                               }
-                               } catch (Exception) {
-                                       // The file system infos were changed while we processed them
-                               }
-                               // Deleted
-                               try {
-                                       bool deleteMatched = true;
-                                       while(deleteMatched) {
-                                               foreach (KeventFileData entry in data.DirEntries.Values) { 
-                                                       if (!File.Exists (entry.fsi.FullName) && !Directory.Exists (entry.fsi.FullName)) {
-                                                               filename = entry.fsi.Name;
-                                                               fa = FileAction.Removed;
-                                                               data.DirEntries.Remove (entry.fsi.FullName);
-                                                               changedFsi = entry.fsi;
-                                                               PostEvent(filename, fsw, fa, changedFsi);
-                                                               break;
+                                       for (var i = 0; i < numEvents; i++) {
+                                               var kevt = out_array [i];
+                                               if ((kevt.flags & EventFlags.Error) == EventFlags.Error)
+                                                       throw new Exception ("kevent error");
+
+                                               if ((kevt.fflags & FilterFlags.VNodeDelete) != 0) {
+                                                       removeQueue.Add (kevt.ident);
+                                                       var data = fdsDict [kevt.ident];
+                                                       PostEvent (data.Path, data.FSW, FileAction.Removed, data.Path);
+                                               } else if (((kevt.fflags & FilterFlags.VNodeRename) != 0) || ((kevt.fflags & FilterFlags.VNodeRevoke) != 0) || ((kevt.fflags & FilterFlags.VNodeWrite) != 0)) {
+                                                       var data = fdsDict [kevt.ident];
+                                                       if (data.IsDirectory && Directory.Exists (data.Path))
+                                                               rescanQueue.Add (kevt.ident);
+
+                                                       if ((kevt.fflags & FilterFlags.VNodeRename) != 0) {
+                                                               var newFilename = GetFilenameFromFd (data.fd);
+                                                               Remove (data.fd);
+                                                               PostEvent (data.Path, data.FSW, FileAction.RenamedNewName, data.Path, newFilename);
+
+                                                               var newEvent = new KeventData {
+                                                                       Path = newFilename,
+                                                                       FileMask = data.FileMask,
+                                                                       FSW = data.FSW,
+                                                                       IncludeSubdirs = data.IncludeSubdirs
+                                                               };
+
+                                                               Add (newEvent, false);
                                                        }
+                                               } else if ((kevt.fflags & FilterFlags.VNodeAttrib) != 0) {
+                                                       var data = fdsDict[kevt.ident];
+                                                       PostEvent (data.Path, data.FSW, FileAction.Modified, data.Path);
                                                }
-                                               deleteMatched = false;
                                        }
-                               } catch (Exception) {
-                                       // The file system infos were changed while we processed them
-                               }
-                               // Added
-                               try {
-                                       foreach (FileSystemInfo fsi in dir.GetFileSystemInfos()) 
-                                               if (!data.DirEntries.ContainsKey (fsi.FullName)) {
-                                                       changedFsi = fsi;
-                                                       filename = fsi.Name;
-                                                       fa = FileAction.Added;
-                                                       data.DirEntries [fsi.FullName] = new KeventFileData(fsi, fsi.LastAccessTime, fsi.LastWriteTime);
-                                                       PostEvent(filename, fsw, fa, changedFsi);
-                                               }
-                               } catch (Exception) {
-                                       // The file system infos were changed while we processed them
+                               } else {
+                                       Thread.Sleep (500);
                                }
-                               
 
+                               firstRun = false;
+                       }
+
+                       lock (this) {
+                               thread = null;
+                               stop = false;
                        }
                }
 
-               private void PostEvent (string filename, FileSystemWatcher fsw, FileAction fa, FileSystemInfo changedFsi) {
+               private void PostEvent (string filename, FileSystemWatcher fsw, FileAction fa, string fullname, string newname = null)
+               {
                        RenamedEventArgs renamed = null;
+
                        if (fa == 0)
                                return;
-                       
-                       if (fsw.IncludeSubdirectories && fa == FileAction.Added) {
-                               if (changedFsi is DirectoryInfo) {
-                                       KeventData newdirdata = new KeventData ();
-                                       newdirdata.FSW = fsw;
-                                       newdirdata.Directory = changedFsi.FullName;
-                                       newdirdata.FileMask = fsw.MangledFilter;
-                                       newdirdata.IncludeSubdirs = fsw.IncludeSubdirectories;
-       
-                                       newdirdata.Enabled = true;
-                                       lock (this) {
-                                               StartMonitoringDirectory (newdirdata);
-                                       }
-                               }
-                       }
-               
-                       if (!fsw.Pattern.IsMatch(filename, true))
-                               return;
+
+                       if (fa == FileAction.RenamedNewName)
+                               renamed = new RenamedEventArgs (WatcherChangeTypes.Renamed, "", newname, fullname);
 
                        lock (fsw) {
-                               if (changedFsi.FullName.StartsWith (fsw.FullPath, StringComparison.Ordinal)) {
-                                       if (fsw.FullPath.EndsWith ("/", StringComparison.Ordinal)) {
-                                               filename = changedFsi.FullName.Substring (fsw.FullPath.Length);
-                                       } else {
-                                               filename = changedFsi.FullName.Substring (fsw.FullPath.Length + 1);
-                                       }
-                               }
                                fsw.DispatchEvents (fa, filename, ref renamed);
                                if (fsw.Waiting) {
                                        fsw.Waiting = false;
@@ -445,6 +471,20 @@ namespace System.IO {
                        }
                }
 
+               private string GetFilenameFromFd (int fd)
+               {
+                       var sb = new StringBuilder (1024);
+
+                       if (fcntl (fd, 50 /* F_GETPATH */, sb) != -1) {
+                               return sb.ToString ();
+                       } else {
+                               return String.Empty;
+                       }
+               }
+
+               [DllImport("libc", EntryPoint="fcntl", CharSet=CharSet.Auto, SetLastError=true)]
+               public static extern int fcntl (int file_names_by_descriptor, int cmd, StringBuilder sb);
+
                [DllImport ("libc")]
                extern static int open(string path, int flags, int mode_t);
                
@@ -455,7 +495,6 @@ namespace System.IO {
                extern static int kqueue();
 
                [DllImport ("libc")]
-               extern static int kevent(int kqueue, ref kevent ev, int nchanges, ref kevent evtlist,  int nevents, ref timespec ts);
+               extern static int kevent(int kq, [In]kevent[] ev, int nchanges, [Out]kevent[] evtlist, int nevents, IntPtr time);
        }
 }
-