From 316f6aea7185d4349df9b970bf5314a0cf789755 Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi <yamamoto@midokura.com>
Date: Wed, 9 Nov 2022 15:55:14 +0900
Subject: [PATCH] group_setupidlefiles: fall back to /dev/null if /dev/console
 is disabled

Tested with a modified nsh on esp32-devkitc.
```
 int main(int argc, FAR char *argv[])
  {
  +  int fd = open("/dev/null", O_RDWR);
  +  _info("hey _info %d\n", fd);
  +  write(1, "hey stdout\n", 11);
  +  write(2, "hey stderr\n", 11);
```
---
 sched/group/group_setupidlefiles.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/sched/group/group_setupidlefiles.c b/sched/group/group_setupidlefiles.c
index 912d2eb57b..eeb6d783a3 100644
--- a/sched/group/group_setupidlefiles.c
+++ b/sched/group/group_setupidlefiles.c
@@ -58,7 +58,7 @@
 
 int group_setupidlefiles(FAR struct task_tcb_s *tcb)
 {
-#ifdef CONFIG_DEV_CONSOLE
+#if defined(CONFIG_DEV_CONSOLE) || defined(CONFIG_DEV_NULL)
   int fd;
 #endif
 
@@ -69,34 +69,47 @@ int group_setupidlefiles(FAR struct task_tcb_s *tcb)
    * descriptor 0.
    */
 
+#if defined(CONFIG_DEV_CONSOLE) || defined(CONFIG_DEV_NULL)
 #ifdef CONFIG_DEV_CONSOLE
   fd = nx_open("/dev/console", O_RDWR);
+#else
+  fd = nx_open("/dev/null", O_RDWR);
+#endif
   if (fd == 0)
     {
-      /* Successfully opened /dev/console as stdin (fd == 0) */
+      /* Successfully opened stdin (fd == 0) */
 
       nx_dup2(0, 1);
       nx_dup2(0, 2);
     }
   else
     {
-      /* We failed to open /dev/console OR for some reason, we opened
+      /* We failed to open stdin OR for some reason, we opened
        * it and got some file descriptor other than 0.
        */
 
       if (fd > 0)
         {
-          sinfo("Open /dev/console fd: %d\n", fd);
+          sinfo("Open stdin fd: %d\n", fd);
           nx_close(fd);
         }
       else
         {
-          serr("ERROR: Failed to open /dev/console: %d\n", fd);
+          serr("ERROR: Failed to open stdin: %d\n", fd);
         }
 
       return -ENFILE;
     }
-#endif
+#else
+  /* This configuration can confuse user programs and libraries.
+   * Eg. a program which opens a file and then prints something to
+   * STDERR_FILENO (2) can end up with something undesirable if the
+   * file descriptor for the file happens to be 2.
+   * It's a common practice to keep 0-2 always open even if they are
+   * /dev/null to avoid that kind of problems. Thus the following warning.
+   */
+#warning file descriptors 0-2 are not opened
+#endif /* defined(CONFIG_DEV_CONSOLE) || defined(CONFIG_DEV_NULL) */
 
   /* Allocate file/socket streams for the TCB */