Add eshell-shell-command and eshell-expand-to-eshell-shell-commandseries/unexpanded-input-v2

* lisp/eshell/esh-mode.el (eshell-shell-command,
eshell-expand-to-eshell-shell-command): Define new functions.
Register eshell-expand-to-eshell-shell-command as a customization
option for eshell-expand-input-functions, and add it by default.
* etc/NEWS:
* doc/misc/eshell.texi: Document the new syntax.

3 files changed, 122 insertions, 1 deletions

diff --git a/doc/misc/eshell.texi b/doc/misc/eshell.texi
index a87dd4308c5..7e9233c09e1 100644
--- a/doc/misc/eshell.texi
+++ b/doc/misc/eshell.texi
@@ -875,6 +875,7 @@ is equivalent to entering the value of @code{var} at the prompt.}
 @menu
 * Dollars Expansion::
 * Globbing::
+* Running Shell Pipelines Natively::
 @end menu
 
 @node Dollars Expansion
@@ -945,6 +946,40 @@ Eshell see the Customize@footnote{@xref{Easy Customization, , , emacs,
 The GNU Emacs Manual}.}
 groups ``eshell-glob'' and ``eshell-pred''.
 
+@node Running Shell Pipelines Natively
+@section Running Shell Pipelines Natively
+When constructing shell pipelines that will move a lot of data, it is
+a good idea to bypass Eshell's own pipelining support and use the
+operating system shell's instead.  This is especially relevant when
+executing commands on a remote machine using Eshell's Tramp
+integration: using the remote shell's pipelining avoids copying the
+data which will flow through the pipeline to local Emacs buffers and
+then right back again.
+
+To quickly construct a command for the native shell, prefix your
+Eshell input with the string @code{||}.  For example,
+
+@example
+|| cat *.ogg | my-cool-decoder >file
+@end example
+
+Executing this command will not copy all the data in the *.ogg files
+into Emacs buffers, as would normally happen with Eshell's own
+pipelining.
+
+As this feature is implemented as an expansion, your original input
+will be replaced with something like this:
+
+@example
+eshell-shell-command \"cat *.ogg | my-cool-decoder >file\"
+@end example
+
+This makes it clear what Eshell is doing, but has the disadvantage of
+being harder to edit, especially for complex pipelines with a lot of
+quoting.  If you would prefer that the @code{||}-prefixed input be
+restored, customize the variable @code{eshell-input-filter-functions}
+to include the function @code{eshell-restore-unexpanded-input}.
+
 @node Input/Output
 @chapter Input/Output
 Since Eshell does not communicate with a terminal like most command
diff --git a/etc/NEWS b/etc/NEWS
index 57fe40c4881..767763098bc 100644
--- a/etc/NEWS
+++ b/etc/NEWS
@@ -749,6 +749,15 @@ the Netscape web browser was released in February, 2008.
 This support has been obsolete since Emacs 25.1.  The final version of
 the Galeon web browser was released in September, 2008.
 
+** Eshell
+
++++
+*** New feature to easily bypass Eshell's own pipelining.
+Prefixing commands with '||' causes them to be executed using the
+operating system shell, which is particularly useful to bypass
+Eshell's own pipelining for pipelines which will move a lot of data.
+See "Running Shell Pipelines Natively" in the Eshell manual.
+
 
 * New Modes and Packages in Emacs 29.1
 
diff --git a/lisp/eshell/esh-mode.el b/lisp/eshell/esh-mode.el
index 87166ef3bf1..04680e4305c 100644
--- a/lisp/eshell/esh-mode.el
+++ b/lisp/eshell/esh-mode.el
@@ -63,6 +63,7 @@
 (require 'esh-cmd)
 (require 'esh-arg)                      ;For eshell-parse-arguments
 (require 'cl-lib)
+(require 'subr-x)
 
 (defgroup eshell-mode nil
   "This module contains code for handling input from the user."
@@ -105,7 +106,8 @@ The input is contained in the region from `eshell-last-input-start' to
   "If t, send any input immediately to a subprocess."
   :type 'boolean)
 
-(defcustom eshell-expand-input-functions nil
+(defcustom eshell-expand-input-functions
+  '(eshell-expand-to-eshell-shell-command)
   "Functions to call before input is parsed.
 Each function is passed two arguments, which bounds the region of the
 current input text."
@@ -671,6 +673,81 @@ newline."
 	       (run-hooks 'eshell-post-command-hook)
 	       (insert-and-inherit input)))))))))
 
+(defun eshell-shell-command (command)
+  "Execute COMMAND using the operating system shell."
+  (throw 'eshell-replace-command
+         (with-connection-local-variables
+	  (eshell-parse-command shell-file-name
+			        (list shell-command-switch command)))))
+
+(defun eshell-expand-to-eshell-shell-command (beg end)
+  "Expand Eshell input starting with '||' to use `eshell-shell-command'.
+
+This is intended to provide a convenient way to bypass Eshell's
+own pipelining which can be slow when executing shell pipelines
+which move a lot of data.  It is also useful to avoid
+roundtripping data when running pipelines on remote hosts.
+
+For example, this function expands the input
+
+    || cat *.ogg | my-cool-decoder >file
+
+to
+
+    eshell-shell-command \"cat *.ogg | my-cool-decoder >file\"
+
+Executing the latter command will not copy all the data in the
+*.ogg files into Emacs buffers, as would normally happen with
+Eshell's own pipelining.
+
+This function tries to extract Eshell-specific redirects and
+avoids passing these to the operating system shell.  For example,
+
+    || foo | bar >>#<buffer scratch>
+
+will be expanded to
+
+    eshell-shell-command \"foo | bar\" >>#<buffer scratch>
+
+This function works well in combination with adding
+`eshell-restore-unexpanded-input' to `eshell-input-filter-functions'."
+  (save-excursion
+    (goto-char beg)
+    (when (looking-at "||\\s-*")
+      (let ((end (copy-marker end)) ; needs to be a marker
+            redirects)
+        ;; drop the ||
+        (delete-region beg (match-end 0))
+        ;; extract Eshell-specific redirects
+        (while (search-forward-regexp "[0-9]?>+&?[0-9]?\\s-*\\S-" nil t)
+          (let ((beg (match-beginning 0)))
+            (forward-char -1) ; start from the redirect target
+            (when-let ((end (cond
+                             ;; this is a redirect to a process or a
+                             ;; buffer
+                             ((looking-at "#<")
+                              (forward-char 1)
+                              (1+ (eshell-find-delimiter ?\< ?\>)))
+                             ;; this is a redirect to a virtual target
+                             ((and (looking-at "/\\S-+")
+                                   (assoc (match-string 0)
+                                          eshell-virtual-targets))
+                              (match-end 0)))))
+              (push (buffer-substring-no-properties beg end) redirects)
+              (delete-region beg end)
+              (just-one-space))))
+        ;; wrap the remaining text and reinstate Eshell redirects
+        (let ((pipeline (string-trim
+                         (buffer-substring-no-properties beg end))))
+          (delete-region beg end)
+          (insert "eshell-shell-command ")
+          (prin1 pipeline (current-buffer))
+          (when redirects
+            (insert " " (string-join (nreverse redirects) " "))))))))
+
+(custom-add-option 'eshell-expand-input-functions
+                   'eshell-expand-to-eshell-shell-command)
+
 (defsubst eshell-kill-new ()
   "Add the last input text to the kill ring."
   (kill-ring-save eshell-last-input-start eshell-last-input-end))