fs/mqueue: Fix the wrong field type in mq_attr as the spec and add the logical judgment to deal with the condition when mq_maxmsg or mq_msgsize is less than zero or equal to zero.

In POSIX standard spec https://pubs.opengroup.org/onlinepubs/7908799/xsh/mqueue.h.html, the field type in mq_attr should be long not size_t. And no logical judgment when mq_maxmsg <= 0 or mq_msgsize <= 0. In this change, i update the field type in mq_attr, and add the missing logical judgment.

Signed-off-by: yangjiao <yangjiao@xiaomi.com>

fs/mqueue/mq_open.c

@@ -191,6 +191,11 @@ static int file_mq_vopen(FAR struct file *mq, FAR const char *mq_name,
 
       mode = va_arg(ap, mode_t);
       attr = va_arg(ap, FAR struct mq_attr *);
+      if (attr->mq_maxmsg <= 0 || attr->mq_msgsize <= 0)
+        {
+          ret = -EINVAL;
+          goto errout;
+        }
     }
 
   mode &= ~umask;

include/mqueue.h

@@ -42,10 +42,10 @@
 
 struct mq_attr
 {
-  size_t         mq_maxmsg;    /* Max number of messages in queue */
-  size_t         mq_msgsize;   /* Max message size */
-  unsigned       mq_flags;     /* Queue flags */
-  size_t         mq_curmsgs;   /* Number of messages currently in queue */
+  long    mq_maxmsg;    /* Max number of messages in queue */
+  long    mq_msgsize;   /* Max message size */
+  long    mq_flags;     /* Queue flags */
+  long    mq_curmsgs;   /* Number of messages currently in queue */
 };
 
 /* Message queue descriptor */