dimonomid
diff --git a/‎.github/workflows/ci-test.yml
Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions b/‎.github/workflows/ci-test.yml
Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions
diff --git a/‎Makefile
Copy file name to clipboard
+10Lines changed: 10 additions & 0 deletions b/‎Makefile
Copy file name to clipboard
+10Lines changed: 10 additions & 0 deletions
diff --git a/‎README.md
Copy file name to clipboard
+4-7Lines changed: 4 additions & 7 deletions b/‎README.md
Copy file name to clipboard
+4-7Lines changed: 4 additions & 7 deletions
diff --git a/‎cmd/nerdlog/app.go
Copy file name to clipboard
+24-6Lines changed: 24 additions & 6 deletions b/‎cmd/nerdlog/app.go
Copy file name to clipboard
+24-6Lines changed: 24 additions & 6 deletions
diff --git a/‎cmd/nerdlog/e2e_testdata/test_scenarios/03_cli_flag_set_numlines/want_screen_01_initial.txt
Copy file name to clipboard
+2-2Lines changed: 2 additions & 2 deletions b/‎cmd/nerdlog/e2e_testdata/test_scenarios/03_cli_flag_set_numlines/want_screen_01_initial.txt
Copy file name to clipboard
+2-2Lines changed: 2 additions & 2 deletions
diff --git a/‎cmd/nerdlog/options.go
Copy file name to clipboard
+39Lines changed: 39 additions & 0 deletions b/‎cmd/nerdlog/options.go
Copy file name to clipboard
+39Lines changed: 39 additions & 0 deletions
diff --git a/‎core/core_test.go
Copy file name to clipboard
+48-11Lines changed: 48 additions & 11 deletions b/‎core/core_test.go
Copy file name to clipboard
+48-11Lines changed: 48 additions & 11 deletions
diff --git a/‎core/core_testdata/test_cases_core/01_simple/test_scenario.yaml
Copy file name to clipboard
+5-3Lines changed: 5 additions & 3 deletions b/‎core/core_testdata/test_cases_core/01_simple/test_scenario.yaml
Copy file name to clipboard
+5-3Lines changed: 5 additions & 3 deletions
diff --git a/‎core/core_testdata/test_cases_core/01_simple/want_conn_info_127.0.0.1_sshbin.txt
Copy file name to clipboard
+25Lines changed: 25 additions & 0 deletions b/‎core/core_testdata/test_cases_core/01_simple/want_conn_info_127.0.0.1_sshbin.txt
Copy file name to clipboard
+25Lines changed: 25 additions & 0 deletions
diff --git a/‎core/core_testdata/test_cases_core/01_simple/want_conn_info_127.0.0.1.txt renamed to ‎core/core_testdata/test_cases_core/01_simple/want_conn_info_127.0.0.1_sshlib.txt
Copy file name to clipboard b/‎core/core_testdata/test_cases_core/01_simple/want_conn_info_127.0.0.1.txt renamed to ‎core/core_testdata/test_cases_core/01_simple/want_conn_info_127.0.0.1_sshlib.txt
Copy file name to clipboard
@@ -116,4 +116,8 @@ jobs:
           eval "$(ssh-agent -s)"
           ssh-add ~/.ssh/id_rsa
 
+          echo "Running tests with ssh-lib"
           NERDLOG_CORE_TEST_HOSTNAME='127.0.0.1' make test ARGS='-run TestCoreScenarios'
+
+          echo "Running tests with ssh-bin"
+          NERDLOG_CORE_TEST_HOSTNAME='127.0.0.1' NERDLOG_CORE_TEST_TRANSPORT_SSH_BIN=1 make test ARGS='-run TestCoreScenarios'
@@ -55,6 +55,9 @@ test-all-variations:
 	@# Rerun core tests using "127.0.0.1" as the hostname, which will use ssh
 	@# transport.
 	NERDLOG_CORE_TEST_HOSTNAME=127.0.0.1 make test ARGS='-run TestCoreScenarios'
+	@# Rerun core tests using "127.0.0.1" as the hostname and external ssh bin.
+	NERDLOG_CORE_TEST_HOSTNAME=127.0.0.1 NERDLOG_CORE_TEST_TRANSPORT_SSH_BIN=1 \
+    make test ARGS='-run TestCoreScenarios'
 
 # Run the tests (without the index-up repetitions), and update all the expected
 # outputs in the repo.
@@ -75,6 +78,13 @@ update-test-expectations:
 	rm -rf /tmp/nerdlog_core_test_output
 	-NERDLOG_CORE_TEST_HOSTNAME=127.0.0.1 make test ARGS='-run TestCoreScenarios'
 	bash util/copy_core_test_results.sh
+	@# Also run core tests with 127.0.0.1 and ssh-bin, to update host-dependent
+	@# payloads for that hostname.
+	rm -rf /tmp/nerdlog_core_test_output
+	-NERDLOG_CORE_TEST_HOSTNAME=127.0.0.1     \
+    NERDLOG_CORE_TEST_TRANSPORT_SSH_BIN=1   \
+    make test ARGS='-run TestCoreScenarios'
+	bash util/copy_core_test_results.sh
 
 bench:
 	# The -run=^$ is needed to avoid running the regular tests as well.
 
@@ -322,16 +322,13 @@ can be done from the Menu too, or using a keyboard shortcut `Alt+Ctrl+R` or
 
 `:version` or `:about` Show version info
 
-`:set option=value` Set option to the new value
-
 `:set option?` Get current value of an option
 
-Currently supported options are:
+`:set option=value` Set option to the new value
+
+See the [full list of supported options here](./docs/options.md).
 
-- `numlines`: the number of log messages loaded from every logstream on every
-  request. Default: 250.
-- `timezone`: the timezone to format the timestamps on the UI. By default,
-  `Local` is used, but you can specify `UTC` or `America/New_York` etc.
+---
 
 `:q[uit]` Quit the app.
 
 
@@ -91,8 +91,9 @@ func newNerdlogApp(
 		params: params,
 
 		options: NewOptionsShared(Options{
-			Timezone:    time.Local,
-			MaxNumLines: 250,
+			Timezone:      time.Local,
+			MaxNumLines:   250,
+			TransportMode: TransportModeSSHLib,
 		}),
 
 		tviewApp: tview.NewApplication(),
@@ -154,8 +155,10 @@ func newNerdlogApp(
 		Logger: logger,
 	})
 
+	useExternalSSH := app.options.GetTransportMode() == TransportModeSSHBin
+
 	// NOTE: initLStreamsManager has to be called _after_ app.mainView is initialized.
-	if err := app.initLStreamsManager(params, "", homeDir, logger); err != nil {
+	if err := app.initLStreamsManager(params, "", useExternalSSH, homeDir, logger); err != nil {
 		return nil, errors.Trace(err)
 	}
 
@@ -179,6 +182,10 @@ func newNerdlogApp(
 		}
 	}
 
+	if len(params.initialOptionSets) > 0 {
+		app.afterUserCmdOrOptionChange()
+	}
+
 	if !params.connectRightAway {
 		app.mainView.params.App.SetFocus(app.mainView.logsTable)
 		app.mainView.queryEditView.Show(params.initialQueryData)
@@ -206,6 +213,7 @@ func (app *nerdlogApp) runTViewApp() error {
 func (app *nerdlogApp) initLStreamsManager(
 	params nerdlogAppParams,
 	initialLStreams string,
+	useExternalSSH bool,
 	homeDir string,
 	logger *log.Logger,
 ) error {
@@ -379,7 +387,8 @@ func (app *nerdlogApp) initLStreamsManager(
 		SSHConfig:        sshConfig,
 		SSHKeys:          params.sshKeys,
 
-		InitialLStreams: initialLStreams,
+		InitialLStreams:       initialLStreams,
+		InitialUseExternalSSH: useExternalSSH,
 
 		ClientID: envUser,
 
@@ -399,12 +408,21 @@ func (app *nerdlogApp) handleCmdLine(cmdCh <-chan cmdWithOpts) {
 				app.cmdLineHistory.Add(cwo.cmd)
 			}
 			app.handleCmd(cwo.cmd)
-			app.mainView.formatTimeRange()
-			app.mainView.formatLogs()
+			app.afterUserCmdOrOptionChange()
 		})
 	}
 }
 
+// afterUserCmdOrOptionChange should be called after user command or option
+// change.
+//
+// TODO: Ideally we need to make it more granular, but good enough for now.
+func (app *nerdlogApp) afterUserCmdOrOptionChange() {
+	app.mainView.formatTimeRange()
+	app.mainView.formatLogs()
+	app.lsman.SetUseExternalSSH(app.options.GetTransportMode() == TransportModeSSHBin)
+}
+
 // printError lets user know that there is an error by printing a simple error
 // message over the command line, sort of like in Vim.
 // Note that if command line is focused atm, the message will not be printed
 
@@ -6,7 +6,7 @@ awk pattern:
   ╔═══════════════════════════════════════════Edit query params═══════════════════════════════════════════╗
   ║                                                                                                       ║
 ti║ Query history:  Back <Ctrl+K>  and  Forth <Ctrl+J>                                                    ║
-  ║                                                                                                       ║
+< ║                                                                                                       ║
   ║ Time range in the format "<time>[ to <time>]", where <time> is either absolute like "Mar27 12:00"     ║
   ║ or relative like "-2h30m" (relative to current time). If the "to" part is omitted,                    ║
   ║ current time is used.                                                                                 ║
@@ -26,5 +26,5 @@ ti║ Query history:  Back <Ctrl+K>  and  Forth <Ctrl+J>
 
 
 
-idle 🖳 00 🖳 00 🖳 00 |
+idle 🖳 00 🖳 00 🖳 00 |                                                                                        -
 
@@ -15,6 +15,20 @@ type Options struct {
 	// MaxNumLines is how many log lines the nerdlog_agent.sh will return at
 	// most. Initially it's set to 250.
 	MaxNumLines int
+
+	TransportMode TransportMode
+}
+
+type TransportMode string
+
+const (
+	TransportModeSSHLib = "ssh-lib"
+	TransportModeSSHBin = "ssh-bin"
+)
+
+var allTransportModes = map[TransportMode]struct{}{
+	TransportModeSSHLib: struct{}{},
+	TransportModeSSHBin: struct{}{},
 }
 
 type OptionsShared struct {
@@ -41,6 +55,12 @@ func (o *OptionsShared) GetMaxNumLines() int {
 	return o.options.MaxNumLines
 }
 
+func (o *OptionsShared) GetTransportMode() TransportMode {
+	o.mtx.Lock()
+	defer o.mtx.Unlock()
+	return o.options.TransportMode
+}
+
 func (o *OptionsShared) GetAll() Options {
 	o.mtx.Lock()
 	defer o.mtx.Unlock()
@@ -100,6 +120,25 @@ var AllOptions = map[string]*OptionMeta{
 	"numlines": {
 		AliasOf: "maxnumlines",
 	}, // }}}
+	"transport": { // {{{
+		Get: func(o *Options) string {
+			return string(o.TransportMode)
+		},
+		Set: func(o *Options, value string) error {
+			if _, ok := allTransportModes[TransportMode(value)]; !ok {
+				slice := make([]string, 0, len(allTransportModes))
+				for v := range allTransportModes {
+					slice = append(slice, string(v))
+				}
+
+				return errors.Errorf("invalid transport mode %q, valid options are: %+v", value, slice)
+			}
+
+			o.TransportMode = TransportMode(value)
+			return nil
+		},
+		Help: "How to connect to remote hosts",
+	}, // }}}
 }
 
 func OptionMetaByName(name string) *OptionMeta {
 
@@ -63,10 +63,16 @@ type CoreTestStep struct {
 }
 
 type CoreTestStepCheckState struct {
-	// WantByHostname is a map from the test hostname (either "localhost",
-	// or overridden by NERDLOG_CORE_TEST_HOSTNAME env var) to the filename
-	// with the expected LStreamsManagerState.
-	WantByHostname map[string]string `yaml:"want_by_hostname"`
+	// WantByHostnameAndTransport is a map from the test hostname-and-transport
+	// string to the filename with the expected LStreamsManagerState.
+	//
+	// The hostname-and-transport string looks like this:
+	//
+	// - "localhost": for localhost, it's just this, since there is only one kind
+	//   of transport
+	// - "127.0.0.1_ssh-lib": for non-localhost hostname without UseExternalSSH
+	// - "127.0.0.1_ssh-bin": for non-localhost hostname with UseExternalSSH
+	WantByHostnameAndTransport map[string]string `yaml:"want_by_hostname_and_transport"`
 }
 
 type CoreTestStepQuery struct {
@@ -195,12 +201,11 @@ func runCoreTestScenario(t *testing.T, tsCtx *coreTestScenarioContext) error {
 				return errors.Annotatef(err, "test step #%d: writing lsm state", i)
 			}
 
-			testHostname := getCoreTestHostname()
-			wantFilename := checkState.WantByHostname[testHostname]
+			hostnameAndTransportKey := manTH.getHostnameAndTransportKey()
+			wantFilename := checkState.WantByHostnameAndTransport[hostnameAndTransportKey]
 
 			if wantFilename == "" {
-				// TODO: maybe we need to fail the tests in this case? not really sure.
-				fmt.Printf("WARNING: no expected lsm state for the hostname %s, skipping that check\n", testHostname)
+				return errors.Errorf("test step #%d: no WantByHostnameAndTransport for %q", i, hostnameAndTransportKey)
 			} else {
 				err = os.WriteFile(filepath.Join(stepOutputDir, "want_lsm_state_filename.txt"), []byte(wantFilename), 0644)
 				if err != nil {
@@ -250,7 +255,9 @@ func runCoreTestScenario(t *testing.T, tsCtx *coreTestScenarioContext) error {
 	}
 
 	fmt.Println("Closing LStreamsManager...")
-	manTH.CloseAndWait()
+	if err := manTH.CloseAndWait(); err != nil {
+		return errors.Trace(err)
+	}
 
 	return nil
 }
@@ -319,6 +326,8 @@ func newLStreamsManagerTestHelper(
 		ClientID:         params.ClientID,
 		UpdatesCh:        updatesCh,
 		Clock:            clockMock,
+
+		InitialUseExternalSSH: os.Getenv("NERDLOG_CORE_TEST_TRANSPORT_SSH_BIN") != "",
 	}
 
 	fmt.Println("Creating LStreamsManager...")
@@ -351,6 +360,21 @@ func getCoreTestHostname() string {
 	return hostname
 }
 
+func (th *LStreamsManagerTestHelper) getHostnameAndTransportKey() string {
+	testHostname := getCoreTestHostname()
+	if testHostname == "localhost" {
+		return testHostname
+	}
+
+	useExternalSSH := th.manager.useExternalSSH
+	transportStr := "ssh-lib"
+	if useExternalSSH {
+		transportStr = "ssh-bin"
+	}
+
+	return fmt.Sprintf("%s_%s", testHostname, transportStr)
+}
+
 func (th *LStreamsManagerTestHelper) run() {
 	for upd := range th.updatesCh {
 		th.applyUpdate(upd)
@@ -437,9 +461,22 @@ func (th *LStreamsManagerTestHelper) WaitNextLogResp() (*LogRespTotal, error) {
 	}
 }
 
-func (th *LStreamsManagerTestHelper) CloseAndWait() {
+func (th *LStreamsManagerTestHelper) CloseAndWait() error {
 	th.manager.Close()
-	th.manager.Wait()
+
+	closedCh := make(chan struct{})
+	go func() {
+		th.manager.Wait()
+		close(closedCh)
+	}()
+
+	select {
+	case <-closedCh:
+		// All good
+		return nil
+	case <-time.After(5 * time.Second):
+		return errors.Errorf("timed out waiting for LStreamsManager to close")
+	}
 }
 
 func formatLogResp(logResp *LogRespTotal) string {
 
@@ -63,8 +63,10 @@ test_steps:
   # for later. For now just moving this step to be the last one.
   - descr: "check connection state"
     check_state:
-      want_by_hostname:
+      want_by_hostname_and_transport:
         # localhost uses local transport
         localhost: want_conn_info_localhost.txt
-        # 127.0.0.1 uses ssh transport
-        127.0.0.1: want_conn_info_127.0.0.1.txt
+        # 127.0.0.1 with ssh-lib transport
+        127.0.0.1_ssh-lib: want_conn_info_127.0.0.1_sshlib.txt
+        # 127.0.0.1 with ssh-bin transport
+        127.0.0.1_ssh-bin: want_conn_info_127.0.0.1_sshbin.txt
@@ -0,0 +1,25 @@
+{
+  "NumLStreams": 1,
+  "LStreamsByState": {
+    "connected_idle": {
+      "testhost-1": {}
+    }
+  },
+  "NumConnected": 1,
+  "NoMatchingLStreams": false,
+  "Connected": true,
+  "Busy": false,
+  "ConnDetailsByLStream": {
+    "testhost-1": {
+      "Messages": [
+        "Trying to connect using external command: \"ssh -o 'BatchMode=yes' 127.0.0.1 /bin/sh\"",
+        "Command started, writing \"echo __CONNECTED__\", waiting for it in stdout",
+        "Got the marker, connected successfully"
+      ],
+      "Err": "",
+      "Connected": true
+    }
+  },
+  "BusyStageByLStream": {},
+  "TearingDown": []
+}