transaction: Add support for Binary conversation

PAM upports binary conversations using private protocols, this
can be handled by C but it's not supported here because we
implicitly convert all the messages to string, and this may lead
to issues when this is not the case (as in binary protocol the
pointer could contain zeros that the GoString conversion would
consider them the end of the message).

So, add another conversation handler implementation that allows
to handle the binary protocol, whose function callback accepts
a pointer to the struct (we can't use bytes as the length is
unknown and may be defined in the header of the pointer itself).

However since the binary prompt is not supported by all the
platforms we need to do a compile-time check to disable it in
case is used when not supported.

1 files changed, 59 insertions, 3 deletions

diff --git a/transaction.go b/transaction.go
index 31a6dbc..96bff63 100644
--- a/transaction.go
+++ b/transaction.go
@@ -1,11 +1,21 @@
 // Package pam provides a wrapper for the PAM application API.
 package pam
 
+//#cgo CFLAGS: -Wall -std=c99
+//#cgo LDFLAGS: -lpam
+//
 //#include <security/pam_appl.h>
 //#include <stdlib.h>
 //#include <stdint.h>
-//#cgo CFLAGS: -Wall -std=c99
-//#cgo LDFLAGS: -lpam
+//
+//#ifdef PAM_BINARY_PROMPT
+//#define BINARY_PROMPT_IS_SUPPORTED 1
+//#else
+//#include <limits.h>
+//#define PAM_BINARY_PROMPT INT_MAX
+//#define BINARY_PROMPT_IS_SUPPORTED 0
+//#endif
+//
 //void init_pam_conv(struct pam_conv *conv, uintptr_t);
 //int pam_start_confdir(const char *service_name, const char *user, const struct pam_conv *pam_conversation, const char *confdir, pam_handle_t **pamh) __attribute__ ((weak));
 //int check_pam_start_confdir(void);
@@ -36,6 +46,9 @@ const (
 	// TextInfo indicates the conversation handler should display some
 	// text.
 	TextInfo = C.PAM_TEXT_INFO
+	// BinaryPrompt indicates the conversation handler that should implement
+	// the private binary protocol
+	BinaryPrompt = C.PAM_BINARY_PROMPT
 )
 
 // ConversationHandler is an interface for objects that can be used as
@@ -47,6 +60,23 @@ type ConversationHandler interface {
 	RespondPAM(Style, string) (string, error)
 }
 
+// BinaryPointer exposes the type used for the data in a binary conversation
+// it represents a pointer to data that is produced by the module and that
+// must be parsed depending on the protocol in use
+type BinaryPointer unsafe.Pointer
+
+// BinaryConversationHandler is an interface for objects that can be used as
+// conversation callbacks during PAM authentication if binary protocol is going
+// to be supported.
+type BinaryConversationHandler interface {
+	ConversationHandler
+	// RespondPAMBinary receives a pointer to the binary message. It's up to
+	// the receiver to parse it according to the protocol specifications.
+	// The function can return a byte array that will passed as pointer back
+	// to the module.
+	RespondPAMBinary(BinaryPointer) ([]byte, error)
+}
+
 // ConversationFunc is an adapter to allow the use of ordinary functions as
 // conversation callbacks.
 type ConversationFunc func(Style, string) (string, error)
@@ -57,14 +87,29 @@ func (f ConversationFunc) RespondPAM(s Style, msg string) (string, error) {
 }
 
 // cbPAMConv is a wrapper for the conversation callback function.
+//
 //export cbPAMConv
 func cbPAMConv(s C.int, msg *C.char, c C.uintptr_t) (*C.char, C.int) {
 	var r string
 	var err error
 	v := cgo.Handle(c).Value()
+	style := Style(s)
 	switch cb := v.(type) {
+	case BinaryConversationHandler:
+		if style == BinaryPrompt {
+			bytes, err := cb.RespondPAMBinary(BinaryPointer(msg))
+			if err != nil {
+				return nil, C.PAM_CONV_ERR
+			}
+			return (*C.char)(C.CBytes(bytes)), C.PAM_SUCCESS
+		} else {
+			r, err = cb.RespondPAM(style, C.GoString(msg))
+		}
 	case ConversationHandler:
-		r, err = cb.RespondPAM(Style(s), C.GoString(msg))
+		if style == BinaryPrompt {
+			return nil, C.PAM_AUTHINFO_UNAVAIL
+		}
+		r, err = cb.RespondPAM(style, C.GoString(msg))
 	}
 	if err != nil {
 		return nil, C.PAM_CONV_ERR
@@ -117,6 +162,12 @@ func StartConfDir(service, user string, handler ConversationHandler, confDir str
 }
 
 func start(service, user string, handler ConversationHandler, confDir string) (*Transaction, error) {
+	switch handler.(type) {
+	case BinaryConversationHandler:
+		if !CheckPamHasBinaryProtocol() {
+			return nil, errors.New("BinaryConversationHandler() was used, but it is not supported by this platform")
+		}
+	}
 	t := &Transaction{
 		conv: &C.struct_pam_conv{},
 		c:    cgo.NewHandle(handler),
@@ -339,3 +390,8 @@ func (t *Transaction) GetEnvList() (map[string]string, error) {
 func CheckPamHasStartConfdir() bool {
 	return C.check_pam_start_confdir() == 0
 }
+
+// CheckPamHasBinaryProtocol return if pam on system supports PAM_BINARY_PROMPT
+func CheckPamHasBinaryProtocol() bool {
+	return C.BINARY_PROMPT_IS_SUPPORTED != 0
+}