[System.IO] Reimplemented much of the kqueue-based file watcher so that watching...
authorCody Russell <cody@jhu.edu>
Sun, 8 Jun 2014 21:46:18 +0000 (16:46 -0500)
committerAlexis Christoforides <alexis@thenull.net>
Tue, 28 Oct 2014 01:20:16 +0000 (21:20 -0400)
Reworked a lot of the internals of the kqueue FileSystemWatcher. There
is now a separate thread and a separate kqueue for each FileSystemWatcher
object.

https://bugzilla.xamarin.com/show_bug.cgi?id=16259

mcs/class/System/System.IO/KeventWatcher.cs

index e673f99739bcaecc9623faa6a277b7c5ca60ef44..2b96de344040d978b619c85325c675b32259d703 100644 (file)
@@ -3,6 +3,7 @@
 //
 // Authors:
 //     Geoff Norton (gnorton@customerdna.com)
+//     Cody Russell (cody@xamarin.com)
 //
 // (c) 2004 Geoff Norton
 // Copyright 2014 Xamarin Inc
@@ -29,6 +30,7 @@
 
 using System;
 using System.Collections;
+using System.Collections.Generic;
 using System.ComponentModel;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -123,13 +125,14 @@ namespace System.IO {
                 TimerNanoSeconds  =   0x00000004,
                 TimerAbsolute     =   0x00000008,
         }
-                       
+
+       [StructLayout(LayoutKind.Sequential)]
        struct kevent : IDisposable {
                public int ident;
                public EventFilter filter;
                public EventFlags flags;
                public FilterFlags fflags;
-               public int data;
+               public IntPtr data;
                public IntPtr udata;
 
                public void Dispose ()
@@ -144,300 +147,238 @@ namespace System.IO {
                public int tv_usec;
        }
 
-       class KeventFileData {
-               public FileSystemInfo fsi;
-               public DateTime LastAccessTime;
-               public DateTime LastWriteTime;
+       class PathData
+       {
+               public string Path;
+               public bool IsDirectory;
+       }
 
-               public KeventFileData(FileSystemInfo fsi, DateTime LastAccessTime, DateTime LastWriteTime) {
-                       this.fsi = fsi;
-                       this.LastAccessTime = LastAccessTime;
-                       this.LastWriteTime = LastWriteTime;
+       class KqueueMonitor : IDisposable
+       {
+               public int Connection
+               {
+                       get { return conn; }
                }
-       }
 
-       class KeventData {
-                public FileSystemWatcher FSW;
-                public string Directory;
-                public string FileMask;
-                public bool IncludeSubdirs;
-                public bool Enabled;
-               public Hashtable DirEntries;
-               public kevent ev;
-        }
+               public KqueueMonitor (FileSystemWatcher fsw)
+               {
+                       this.fsw = fsw;
+                       this.conn = -1;
+               }
 
-       class KeventWatcher : IFileWatcher
-       {
-               static bool failed;
-               static KeventWatcher instance;
-               static Hashtable watches;
-               static Hashtable requests;
-               static Thread thread;
-               static int conn;
-               static bool stop;
-               
-               private KeventWatcher ()
+               public void Dispose ()
                {
+                       Stop ();
                }
-               
-               // Locked by caller
-               public static bool GetInstance (out IFileWatcher watcher)
+
+               public void Start ()
                {
-                       if (failed == true) {
-                               watcher = null;
-                               return false;
-                       }
+                       conn = kqueue ();
 
-                       if (instance != null) {
-                               watcher = instance;
-                               return true;
+                       if (thread == null) {
+                               thread = new Thread (new ThreadStart (Monitor));
+                               thread.IsBackground = true;
+                               thread.Start ();
                        }
 
-                       watches = Hashtable.Synchronized (new Hashtable ());
-                       requests = Hashtable.Synchronized (new Hashtable ());
-                       conn = kqueue();
-                       if (conn == -1) {
-                               failed = true;
-                               watcher = null;
-                               return false;
-                       }
+                       var pathData = Add (fsw.FullPath);
 
-                       instance = new KeventWatcher ();
-                       watcher = instance;
-                       return true;
+                       Scan (pathData);
                }
-               
-               public void StartDispatching (FileSystemWatcher fsw)
+
+               public void Stop ()
                {
-                       KeventData data;
-                       lock (this) {
-                               if (thread == null) {
-                                       thread = new Thread (new ThreadStart (Monitor));
-                                       thread.IsBackground = true;
-                                       thread.Start ();
-                               }
+                       stop = true;
 
-                               data = (KeventData) watches [fsw];
-                       }
+                       if (thread != null)
+                               thread.Interrupt ();
 
-                       if (data == null) {
-                               data = new KeventData ();
-                               data.FSW = fsw;
-                               data.Directory = fsw.FullPath;
-                               data.FileMask = fsw.MangledFilter;
-                               data.IncludeSubdirs = fsw.IncludeSubdirectories;
-
-                               data.Enabled = true;
-                               lock (this) {
-                                       StartMonitoringDirectory (data);
-                                       watches [fsw] = data;
-                                       stop = false;
-                               }
-                       }
+                       if (conn != -1)
+                               close (conn);
+
+                       conn = -1;
                }
 
-               static void StartMonitoringDirectory (KeventData data)
+               private PathData FindPath (string path)
                {
-                       DirectoryInfo dir = new DirectoryInfo (data.Directory);
-                       if(data.DirEntries == null) {
-                               data.DirEntries = new Hashtable();
-                               foreach (FileSystemInfo fsi in dir.GetFileSystemInfos() ) 
-                                       data.DirEntries.Add(fsi.FullName, new KeventFileData(fsi, fsi.LastAccessTime, fsi.LastWriteTime));
+                       foreach (KeyValuePair<PathData, int> kv in paths) {
+                               if (kv.Key.Path == path)
+                                       return kv.Key;
                        }
 
-                       int fd = open(data.Directory, 0, 0);
-                       kevent ev = new kevent();
-                       ev.udata = IntPtr.Zero;
-                       timespec nullts = new timespec();
-                       nullts.tv_sec = 0;
-                       nullts.tv_usec = 0;
-                       if (fd > 0) {
-                               ev.ident = fd;
-                               ev.filter = EventFilter.Vnode;
-                               ev.flags = EventFlags.Add | EventFlags.Enable | EventFlags.OneShot;
-                               ev.fflags = // 20 | 2 | 1 | 8;
-                                       FilterFlags.VNodeDelete |
-                                       FilterFlags.VNodeWrite |
-                                       FilterFlags.VNodeAttrib |
-                                       // The following two values are the equivalent of the original value "20", but we suspect the original author meant
-                                       // 0x20, we will review later with some test cases
-                                       FilterFlags.VNodeLink |
-                                       FilterFlags.VNodeExtend;
-                               ev.data = 0;
-                               ev.udata = Marshal.StringToHGlobalAuto (data.Directory);
-                               kevent outev = new kevent();
-                               outev.udata = IntPtr.Zero;
-                               kevent (conn, ref ev, 1, ref outev, 0, ref nullts);
-                               data.ev = ev;
-                               requests [fd] = data;
-                       }
-                       
-                       if (!data.IncludeSubdirs)
-                               return;
-
+                       return null;
                }
 
-               public void StopDispatching (FileSystemWatcher fsw)
+               private PathData FindPath (int fd)
                {
-                       KeventData data;
-                       lock (this) {
-                               data = (KeventData) watches [fsw];
-                               if (data == null)
-                                       return;
-
-                               StopMonitoringDirectory (data);
-                               watches.Remove (fsw);
-                               if (watches.Count == 0)
-                                       stop = true;
-
-                               if (!data.IncludeSubdirs)
-                                       return;
-
+                       foreach (KeyValuePair<PathData, int> kv in paths) {
+                               if (kv.Value == fd)
+                                       return kv.Key;
                        }
-               }
 
-               static void StopMonitoringDirectory (KeventData data)
-               {
-                       close(data.ev.ident);
+                       return null;
                }
 
-               void Monitor ()
+               private void Monitor ()
                {
-               
+                       bool firstRun = true;
+
                        while (!stop) {
-                               kevent ev = new kevent();
-                               ev.udata = IntPtr.Zero;
-                               kevent nullev = new kevent();
-                               nullev.udata = IntPtr.Zero;
-                               timespec ts = new timespec();
-                               ts.tv_sec = 0;
-                               ts.tv_usec = 0;
-                               int haveEvents;
-                               lock (this) {
-                                       haveEvents = kevent (conn, ref nullev, 0, ref ev, 1, ref ts);
+                               removeQueue.ForEach (Remove);
+
+                               var changes = new List<kevent> ();
+                               var outEvents = new List<kevent> ();
+
+                               rescanQueue.ForEach (fd => {
+                                       var path = FindPath (fd);
+                                       Scan (path, !firstRun);
+                               });
+                               rescanQueue.Clear ();
+
+                               foreach (KeyValuePair<PathData, int> kv in paths) {
+                                       var change = new kevent {
+                                               ident  = kv.Value,
+                                               filter = EventFilter.Vnode,
+                                               flags  = EventFlags.Add | EventFlags.Enable | EventFlags.Clear,
+                                               fflags = FilterFlags.VNodeDelete | FilterFlags.VNodeExtend |
+                                                        FilterFlags.VNodeRename | FilterFlags.VNodeAttrib |
+                                                        FilterFlags.VNodeLink | FilterFlags.VNodeRevoke |
+                                                        FilterFlags.VNodeWrite,
+                                               data   = IntPtr.Zero,
+                                               udata  = IntPtr.Zero
+                                       };
+
+                                       changes.Add (change);
+                                       outEvents.Add (new kevent ());
                                }
 
-                               if (haveEvents > 0) {
-                                       // Restart monitoring
-                                       KeventData data = (KeventData) requests [ev.ident];
-                                       StopMonitoringDirectory (data);
-                                       StartMonitoringDirectory (data);
-                                       ProcessEvent (ev);
+                               if (changes.Count > 0) {
+                                       var outArray = outEvents.ToArray ();
+                                       var changesArray = changes.ToArray ();
+                                       int numEvents = kevent (conn, changesArray, changesArray.Length, outArray, outArray.Length, IntPtr.Zero);
+
+                                       for (var i = 0; i < numEvents; i++) {
+                                               var kevt = outArray [i];
+                                               var pathData = FindPath (kevt.ident);
+
+                                               if ((kevt.fflags & FilterFlags.VNodeDelete) != 0) {
+                                                       removeQueue.Add (kevt.ident);
+                                                       PostEvent (FileAction.Removed, pathData.Path);
+                                               } else if (((kevt.fflags & FilterFlags.VNodeRename) != 0) || ((kevt.fflags & FilterFlags.VNodeRevoke) != 0) || ((kevt.fflags & FilterFlags.VNodeWrite) != 0)) {
+                                                       if (pathData.IsDirectory && Directory.Exists (pathData.Path))
+                                                               rescanQueue.Add (kevt.ident);
+
+                                                       if ((kevt.fflags & FilterFlags.VNodeRename) != 0) {
+                                                               var fd = paths [pathData];
+                                                               var newFilename = GetFilenameFromFd (fd);
+                                                               var oldFilename = pathData.Path;
+
+                                                               Remove (pathData);
+                                                               PostEvent (FileAction.RenamedNewName, oldFilename, newFilename);
+
+                                                               Add (newFilename, false);
+                                                       }
+                                               } else if ((kevt.fflags & FilterFlags.VNodeAttrib) != 0) {
+                                                       PostEvent (FileAction.Modified, pathData.Path);
+                                               }
+                                       }
                                } else {
-                                       System.Threading.Thread.Sleep (500);
+                                       Thread.Sleep (500);
                                }
-                       }
 
-                       lock (this) {
-                               thread = null;
-                               stop = false;
+                               firstRun = false;
                        }
                }
 
-               void ProcessEvent (kevent ev)
+               private PathData Add (string path, bool postEvents = false)
                {
-                       lock (this) {
-                               KeventData data = (KeventData) requests [ev.ident];
-                               if (!data.Enabled)
-                                       return;
-
-                               FileSystemWatcher fsw;
-                               string filename = "";
-
-                               fsw = data.FSW;
-                               FileAction fa = 0;
-                               DirectoryInfo dir = new DirectoryInfo (data.Directory);
-                               FileSystemInfo changedFsi = null;
-
-                               try {
-                                       foreach (FileSystemInfo fsi in dir.GetFileSystemInfos() )
-                                               if (data.DirEntries.ContainsKey (fsi.FullName) && (fsi is FileInfo)) {
-                                                       KeventFileData entry = (KeventFileData) data.DirEntries [fsi.FullName];
-                                                       if (entry.LastWriteTime != fsi.LastWriteTime) {
-                                                               filename = fsi.Name;
-                                                               fa = FileAction.Modified;
-                                                               data.DirEntries [fsi.FullName] = new KeventFileData(fsi, fsi.LastAccessTime, fsi.LastWriteTime);
-                                                               if (fsw.IncludeSubdirectories && fsi is DirectoryInfo) {
-                                                                       data.Directory = filename;
-                                                                       requests [ev.ident] = data;
-                                                                       ProcessEvent(ev);
-                                                               }
-                                                               changedFsi = fsi;
-                                                               PostEvent(filename, fsw, fa, changedFsi);
-                                                       }
-                                               }
-                               } catch (Exception) {
-                                       // The file system infos were changed while we processed them
-                               }
-                               // Deleted
-                               try {
-                                       bool deleteMatched = true;
-                                       while(deleteMatched) {
-                                               foreach (KeventFileData entry in data.DirEntries.Values) { 
-                                                       if (!File.Exists (entry.fsi.FullName) && !Directory.Exists (entry.fsi.FullName)) {
-                                                               filename = entry.fsi.Name;
-                                                               fa = FileAction.Removed;
-                                                               data.DirEntries.Remove (entry.fsi.FullName);
-                                                               changedFsi = entry.fsi;
-                                                               PostEvent(filename, fsw, fa, changedFsi);
-                                                               break;
-                                                       }
-                                               }
-                                               deleteMatched = false;
-                                       }
-                               } catch (Exception) {
-                                       // The file system infos were changed while we processed them
-                               }
-                               // Added
-                               try {
-                                       foreach (FileSystemInfo fsi in dir.GetFileSystemInfos()) 
-                                               if (!data.DirEntries.ContainsKey (fsi.FullName)) {
-                                                       changedFsi = fsi;
-                                                       filename = fsi.Name;
-                                                       fa = FileAction.Added;
-                                                       data.DirEntries [fsi.FullName] = new KeventFileData(fsi, fsi.LastAccessTime, fsi.LastWriteTime);
-                                                       PostEvent(filename, fsw, fa, changedFsi);
-                                               }
-                               } catch (Exception) {
-                                       // The file system infos were changed while we processed them
-                               }
-                               
+                       var fd = open (path, O_EVTONLY, 0);
+
+                       if (fd == -1)
+                               return null;
 
+                       var attrs = File.GetAttributes (path);
+                       bool isDir = false;
+                       if ((attrs & FileAttributes.Directory) == FileAttributes.Directory)
+                               isDir = true;
+
+                       var pathData = new PathData {
+                               Path = path,
+                               IsDirectory = isDir
+                       };
+
+                       if (FindPath (path) == null) {
+                               paths.Add (pathData, fd);
+
+                               if (postEvents)
+                                       PostEvent (FileAction.Added, path);
                        }
+
+                       return pathData;
                }
 
-               private void PostEvent (string filename, FileSystemWatcher fsw, FileAction fa, FileSystemInfo changedFsi) {
-                       RenamedEventArgs renamed = null;
-                       if (fa == 0)
+               private void Remove (int fd)
+               {
+                       var path = FindPath (fd);
+                       paths.Remove (path);
+                       removeQueue.Remove (fd);
+
+                       close (fd);
+               }
+
+               private void Remove (PathData pathData)
+               {
+                       var fd = paths [pathData];
+                       paths.Remove (pathData);
+                       removeQueue.Remove (fd);
+
+                       close (fd);
+               }
+
+               private void Scan (PathData pathData, bool postEvents = false)
+               {
+                       var path = pathData.Path;
+
+                       Add (path, postEvents);
+
+                       if (!fsw.IncludeSubdirectories)
                                return;
-                       
-                       if (fsw.IncludeSubdirectories && fa == FileAction.Added) {
-                               if (changedFsi is DirectoryInfo) {
-                                       KeventData newdirdata = new KeventData ();
-                                       newdirdata.FSW = fsw;
-                                       newdirdata.Directory = changedFsi.FullName;
-                                       newdirdata.FileMask = fsw.MangledFilter;
-                                       newdirdata.IncludeSubdirs = fsw.IncludeSubdirectories;
-       
-                                       newdirdata.Enabled = true;
-                                       lock (this) {
-                                               StartMonitoringDirectory (newdirdata);
+
+                       var attrs = File.GetAttributes (path);
+                       if ((attrs & FileAttributes.Directory) == FileAttributes.Directory) {
+                               var dirsToProcess = new List<string> ();
+                               dirsToProcess.Add (path);
+
+                               while (dirsToProcess.Count > 0) {
+                                       var tmp = dirsToProcess [0];
+                                       dirsToProcess.RemoveAt (0);
+
+                                       var info = new DirectoryInfo (tmp);
+                                       foreach (var fsi in info.GetFileSystemInfos ()) {
+                                               if (Add (fsi.FullName, postEvents) == null)
+                                                       continue;
+
+                                               var childAttrs = File.GetAttributes (fsi.FullName);
+                                               if ((childAttrs & FileAttributes.Directory) == FileAttributes.Directory)
+                                                       dirsToProcess.Add (fsi.FullName);
                                        }
                                }
                        }
-               
-                       if (!fsw.Pattern.IsMatch(filename, true))
+               }
+
+               private void PostEvent (FileAction action, string path, string newPath = null)
+               {
+                       RenamedEventArgs renamed = null;
+
+                       if (action == 0)
                                return;
 
+                       if (action == FileAction.RenamedNewName)
+                               renamed = new RenamedEventArgs (WatcherChangeTypes.Renamed, "", newPath, path);
+
                        lock (fsw) {
-                               if (changedFsi.FullName.StartsWith (fsw.FullPath, StringComparison.Ordinal)) {
-                                       if (fsw.FullPath.EndsWith ("/", StringComparison.Ordinal)) {
-                                               filename = changedFsi.FullName.Substring (fsw.FullPath.Length);
-                                       } else {
-                                               filename = changedFsi.FullName.Substring (fsw.FullPath.Length + 1);
-                                       }
-                               }
-                               fsw.DispatchEvents (fa, filename, ref renamed);
+                               fsw.DispatchEvents (action, path, ref renamed);
+
                                if (fsw.Waiting) {
                                        fsw.Waiting = false;
                                        System.Threading.Monitor.PulseAll (fsw);
@@ -445,17 +386,109 @@ namespace System.IO {
                        }
                }
 
+               private string GetFilenameFromFd (int fd)
+               {
+                       var sb = new StringBuilder (1024);
+
+                       if (fcntl (fd, F_GETPATH, sb) != -1)
+                               return sb.ToString ();
+                       else
+                               return String.Empty;
+               }
+
+               private const int O_EVTONLY = 0x8000;
+               private const int F_GETPATH = 50;
+               private FileSystemWatcher fsw;
+               private int conn;
+               private Thread thread;
+               private bool stop;
+               private readonly List<int> removeQueue = new List<int> ();
+               private readonly List<int> rescanQueue = new List<int> ();
+               private readonly Dictionary<PathData, int> paths = new Dictionary<PathData, int> ();
+
+               [DllImport ("libc", EntryPoint="fcntl", CharSet=CharSet.Auto, SetLastError=true)]
+               static extern int fcntl (int file_names_by_descriptor, int cmd, StringBuilder sb);
+
                [DllImport ("libc")]
-               extern static int open(string path, int flags, int mode_t);
-               
+               extern static int open (string path, int flags, int mode_t);
+
+               [DllImport ("libc")]
+               extern static int close (int fd);
+
+               [DllImport ("libc")]
+               extern static int kqueue ();
+
                [DllImport ("libc")]
-               extern static int close(int fd);
+               extern static int kevent(int kq, [In]kevent[] ev, int nchanges, [Out]kevent[] evtlist, int nevents, IntPtr time);
+       }
+
+       class KeventWatcher : IFileWatcher
+       {
+               static bool failed;
+               static KeventWatcher instance;
+               static Hashtable watches;  // <FileSystemWatcher, KqueueMonitor>
+
+               private KeventWatcher ()
+               {
+               }
+
+               // Locked by caller
+               public static bool GetInstance (out IFileWatcher watcher)
+               {
+                       if (failed == true) {
+                               watcher = null;
+                               return false;
+                       }
+
+                       if (instance != null) {
+                               watcher = instance;
+                               return true;
+                       }
+
+                       watches = Hashtable.Synchronized (new Hashtable ());
+                       var conn = kqueue();
+                       if (conn == -1) {
+                               failed = true;
+                               watcher = null;
+                               return false;
+                       }
+                       close (conn);
+
+                       instance = new KeventWatcher ();
+                       watcher = instance;
+                       return true;
+               }
+
+               public void StartDispatching (FileSystemWatcher fsw)
+               {
+                       KqueueMonitor monitor;
+
+                       if (watches.ContainsKey (fsw)) {
+                               monitor = (KqueueMonitor)watches [fsw];
+                       } else {
+                               monitor = new KqueueMonitor (fsw);
+                       }
+
+                       watches.Add (fsw, monitor);
+
+                       monitor.Start ();
+               }
+
+               public void StopDispatching (FileSystemWatcher fsw)
+               {
+                       KqueueMonitor monitor = (KqueueMonitor)watches [fsw];
+                       if (monitor == null)
+                               return;
+
+                       monitor.Stop ();
+               }
+
 
                [DllImport ("libc")]
-               extern static int kqueue();
+               extern static int close (int fd);
 
                [DllImport ("libc")]
-               extern static int kevent(int kqueue, ref kevent ev, int nchanges, ref kevent evtlist,  int nevents, ref timespec ts);
+               extern static int kqueue ();
        }
 }