From fcfa8853e41ca04714a7aa28a783e2804c184375 Mon Sep 17 00:00:00 2001 From: Justin Keogh Date: Thu, 7 Apr 2022 05:58:56 +0000 Subject: [utils] locked_file: Do not truncate files before locking (#2994) Authored by: jakeogh, pukkandan --- yt_dlp/utils.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'yt_dlp/utils.py') diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 87dd04e23..66c3da4c8 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -2222,10 +2222,23 @@ class locked_file(object): locked = False def __init__(self, filename, mode, block=True, encoding=None): - assert mode in {'r', 'rb', 'a', 'ab', 'w', 'wb'} - self.f = open(filename, mode, encoding=encoding) - self.mode = mode - self.block = block + if mode not in {'r', 'rb', 'a', 'ab', 'w', 'wb'}: + raise NotImplementedError(mode) + self.mode, self.block = mode, block + + writable = any(f in mode for f in 'wax+') + readable = any(f in mode for f in 'r+') + flags = functools.reduce(operator.ior, ( + getattr(os, 'O_CLOEXEC', 0), # UNIX only + getattr(os, 'O_BINARY', 0), # Windows only + getattr(os, 'O_NOINHERIT', 0), # Windows only + os.O_CREAT if writable else 0, # O_TRUNC only after locking + os.O_APPEND if 'a' in mode else 0, + os.O_EXCL if 'x' in mode else 0, + os.O_RDONLY if not writable else os.O_RDWR if readable else os.O_WRONLY, + )) + + self.f = os.fdopen(os.open(filename, flags), mode, encoding=encoding) def __enter__(self): exclusive = 'r' not in self.mode @@ -2235,6 +2248,8 @@ class locked_file(object): except IOError: self.f.close() raise + if 'w' in self.mode: + self.f.truncate() return self def unlock(self): -- cgit v1.2.3 From b63837bce0b104b1f72f2ebb6c0d05080cf2a607 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 7 Apr 2022 12:00:58 +0530 Subject: [utils] locked_file: Fix non-blocking non-exclusive lock --- yt_dlp/utils.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'yt_dlp/utils.py') diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 66c3da4c8..02b5ae2ee 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -2190,18 +2190,15 @@ else: import fcntl def _lock_file(f, exclusive, block): + flags = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + if not block: + flags |= fcntl.LOCK_NB try: - fcntl.flock(f, - fcntl.LOCK_SH if not exclusive - else fcntl.LOCK_EX if block - else fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.flock(f, flags) except BlockingIOError: raise except OSError: # AOSP does not have flock() - fcntl.lockf(f, - fcntl.LOCK_SH if not exclusive - else fcntl.LOCK_EX if block - else fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.lockf(f, flags) def _unlock_file(f): try: -- cgit v1.2.3 From b506289fe205cc2f3488f72c826034465cef2d0c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 7 Apr 2022 11:30:46 +0530 Subject: [test] Add `test_locked_file` --- yt_dlp/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'yt_dlp/utils.py') diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 02b5ae2ee..84b2603df 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -684,8 +684,9 @@ def sanitize_open(filename, open_mode): try: try: if sys.platform == 'win32': - # FIXME: Windows only has mandatory locking which also locks the file from being read. - # So for now, don't lock the file on windows. Ref: https://github.com/yt-dlp/yt-dlp/issues/3124 + # FIXME: An exclusive lock also locks the file from being read. + # Since windows locks are mandatory, don't lock the file on windows (for now). + # Ref: https://github.com/yt-dlp/yt-dlp/issues/3124 raise LockingUnsupportedError() stream = locked_file(filename, open_mode, block=False).__enter__() except LockingUnsupportedError: -- cgit v1.2.3 From 4abea8ca0af0773db9fb2372b272d497bd77b207 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 9 Apr 2022 10:11:25 +0530 Subject: [utils] `sanitize_path`: Fix when path is empty string --- yt_dlp/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/utils.py') diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 84b2603df..ba9566cab 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -778,7 +778,7 @@ def sanitize_path(s, force=False): for path_part in norm_path] if drive_or_unc: sanitized_path.insert(0, drive_or_unc + os.path.sep) - elif force and s[0] == os.path.sep: + elif force and s and s[0] == os.path.sep: sanitized_path.insert(0, os.path.sep) return os.path.join(*sanitized_path) -- cgit v1.2.3 From 98804d034d04d21cbeb8cd43d1e1d90f1cdae836 Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Sun, 10 Apr 2022 01:23:27 +0900 Subject: [utils] locked_file: Do not give executable bits for newly created files Authored by: Lesmiscore --- yt_dlp/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/utils.py') diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index ba9566cab..14dbbf59f 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -2236,7 +2236,7 @@ class locked_file(object): os.O_RDONLY if not writable else os.O_RDWR if readable else os.O_WRONLY, )) - self.f = os.fdopen(os.open(filename, flags), mode, encoding=encoding) + self.f = os.fdopen(os.open(filename, flags, 0o666), mode, encoding=encoding) def __enter__(self): exclusive = 'r' not in self.mode -- cgit v1.2.3