serial: Call set_custom_baudrate() thrice

Call the function before tcsetattr() settings are known, then again
with settings prepared but not yet applied and finally a third time
after tcsetattr().

Darwin support needs this change; there custom_baud code must be
called to modify the settings passed to tcsetattr() and then again
after tcsetattr() returns.

The change should be non-functional on all currently supported systems;
current code calls set_custom_baudrate() before any tcsetattr()
settings are prepared, so we have three stages in total.

This change originates from discussion of the macOS patch proposed by
Denis Ahrens in https://review.coreboot.org/c/flashrom/+/67822

Change-Id: I40cc443cfb7bf6b212b31826d437b898cc13c427
Signed-off-by: Peter Stuge <peter@stuge.se>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/70569
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Thomas Heijligen <src@posteo.de>

custom_baud.c

@@ -19,7 +19,7 @@
 #include "custom_baud.h"
 
 /* Stub, should not get called. */
-int set_custom_baudrate(int fd, unsigned int baud)
+int set_custom_baudrate(int fd, unsigned int baud, const enum custom_baud_stage stage, void *tio_wanted)
 {
 	errno = ENOSYS; /* Hoping "Function not supported" will make you look here. */
 	return -1;

custom_baud_linux.c

@@ -29,9 +29,13 @@
  * for more info.
  */
 
-int set_custom_baudrate(int fd, unsigned int baud)
+int set_custom_baudrate(int fd, unsigned int baud, const enum custom_baud_stage stage, void *tio_wanted)
 {
 	struct termios2 tio;
+
+	if (stage != BEFORE_FLAGS)
+		return 0;
+
 	if (ioctl(fd, TCGETS2, &tio)) {
 		return -1;
 	}

include/custom_baud.h

@@ -22,7 +22,13 @@ struct baudentry {
 	unsigned int baud;
 };
 
-int set_custom_baudrate(int fd, unsigned int baud);
+enum custom_baud_stage {
+	BEFORE_FLAGS = 0,
+	WITH_FLAGS,
+	AFTER_FLAGS
+};
+
+int set_custom_baudrate(int fd, unsigned int baud, const enum custom_baud_stage stage, void *tio_wanted);
 
 /* Returns 1 if non-exact rate would be used, and setting a custom rate is supported.
    The baudtable must be in ascending order and terminated with a 0-baud entry. */

serial.c

@@ -179,6 +179,7 @@ int serialport_config(fdtype fd, int baud)
 	}
 	msg_pdbg("Baud rate is %ld.\n", dcb.BaudRate);
 #else
+	int custom_baud = (baud >= 0 && use_custom_baud(baud, sp_baudtable));
 	struct termios wanted, observed;
 	if (tcgetattr(fd, &observed) != 0) {
 		msg_perr_strerror("Could not fetch original serial port configuration: ");
@@ -186,8 +187,8 @@ int serialport_config(fdtype fd, int baud)
 	}
 	wanted = observed;
 	if (baud >= 0) {
-		if (use_custom_baud(baud, sp_baudtable)) {
+		if (custom_baud) {
-			if (set_custom_baudrate(fd, baud)) {
+			if (set_custom_baudrate(fd, baud, BEFORE_FLAGS, NULL)) {
 				msg_perr_strerror("Could not set custom baudrate: ");
 				return 1;
 			}
@@ -198,7 +199,6 @@ int serialport_config(fdtype fd, int baud)
 				msg_perr_strerror("Could not fetch serial port configuration: ");
 				return 1;
 			}
-			msg_pdbg("Using custom baud rate.\n");
 		} else {
 			const struct baudentry *entry = round_baud(baud);
 			if (cfsetispeed(&wanted, entry->flag) != 0 || cfsetospeed(&wanted, entry->flag) != 0) {
@@ -212,6 +212,10 @@ int serialport_config(fdtype fd, int baud)
 	wanted.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG | IEXTEN);
 	wanted.c_iflag &= ~(IXON | IXOFF | IXANY | ICRNL | IGNCR | INLCR);
 	wanted.c_oflag &= ~OPOST;
+	if (custom_baud && set_custom_baudrate(fd, baud, WITH_FLAGS, &wanted)) {
+		msg_perr_strerror("Could not set custom baudrate: ");
+		return 1;
+	}
 	if (tcsetattr(fd, TCSANOW, &wanted) != 0) {
 		msg_perr_strerror("Could not change serial port configuration: ");
 		return 1;
@@ -236,6 +240,13 @@ int serialport_config(fdtype fd, int baud)
 			 (long)observed.c_oflag, (long)wanted.c_oflag
 			);
 	}
+	if (custom_baud) {
+		if (set_custom_baudrate(fd, baud, AFTER_FLAGS, &wanted)) {
+			msg_perr_strerror("Could not set custom baudrate: ");
+			return 1;
+		}
+		msg_pdbg("Using custom baud rate.\n");
+	}
 	if (cfgetispeed(&observed) != cfgetispeed(&wanted) ||
 	    cfgetospeed(&observed) != cfgetospeed(&wanted)) {
 		msg_pwarn("Could not set baud rates exactly.\n");