Bug 4354 - Encryption may cause a deadlock.
: Encryption may cause a deadlock.
Status: NEW
: XIO
GlobusIO
: 4.0.1
: PC Linux
: P3 normal
: ---
Assigned To:
:
:
:
:
  Show dependency treegraph
 
Reported: 2006-04-18 01:26 by
Modified: 2006-08-17 22:49 (History)


Attachments
myprogram.sh (34.52 KB, text/plain)
2006-04-18 01:39, ASOU Masato
Details
Newer client.c (6.20 KB, text/plain)
2006-05-19 00:05, ASOU Masato
Details
My program, it was added authentication only. (37.18 KB, text/plain)
2006-08-17 22:49, ASOU Masato
Details


Note

You need to log in before you can comment on or make changes to this bug.


Description From 2006-04-18 01:26:04
I am useing Red Hat Linux 7.3 on Pentium III and Globus toolkit 4.0.1.

    % uname -mrsv
    Linux 2.4.20-30.8.legacysmp #1 SMP Fri Feb 20 17:13:00 PST 2004 i686
    % globus-version
    4.0.1

If encryption is applied to a communication, a deadlock may occur.
The summary of my program is show below:

    1.A client calls globus_io_write() twice and transmits N bytes of data.
    2.A server calls globus_io_read() twice and receives N bytes of data. 
    3.A server calls globus_io_write() twice and transmits N bytes of data.
    4.A client calls globus_io_read() twice and receives N bytes of data. 

[Server side]

    main()
    {
        globus_io_register_listen(acceptCallback);
    }

    acceptCallback()
    {
        unsigned char buf[NBYTES];
    int nBytes = NBYTES / 2;

        globus_io_tcp_accept();
        globus_io_read(&buf[0], nBytes);
        globus_io_read(&buf[nBytes], nBytes);
        globus_io_write(&buf[0], nBytes);
        globus_io_write(&buf[nBytes], nBytes);
    }

[Client side]

    main()
    {
        unsigned char buf[NBYTES];
    int nBytes = NBYTES / 2;

        globus_io_register_select(receiveCallback);
        globus_io_write(&buf[0], nBytes);
        globus_io_write(&buf[nBytes], nBytes);
    }

    receiveCallback()
    {
        unsigned char buf[NBYTES];
    int nBytes = NBYTES / 2;

        globus_io_read(&buf[0], nBytes);
        globus_io_read(&buf[nBytes], nBytes);
    }

This program works fine, when encryption is not applied. However,
deadlock will occur when encryption is applied.
When I investigate, receiveCallback() is called immediately after the
call of first globus_io_write() at client side. Correctly,
receiveCallback() is called from globus_io_write().

[gdb backtrace]

    Current directory is
~/prj/grid-soft/ninfg-v4/test/experimental/bug251-GT3freeze/
    GNU gdb Red Hat Linux (5.1.90CVS-5)
    Copyright 2002 Free Software Foundation, Inc.
    GDB is free software, covered by the GNU General Public License, and you
are
    welcome to change it and/or distribute copies of it under certain
conditions.
    Type "show copying" to see the conditions.
    There is absolutely no warranty for GDB.  Type "show warranty" for details.
    This GDB was configured as "i386-redhat-linux"...
    (gdb) run --crypt=SSL localhost 10000 100
    Starting program:
/home/asou/prj/grid-soft/ninfg-v4/test/experimental/bug251-GT3freeze/client
--crypt=SSL localhost 10000 100

    Program received signal SIGINT, Interrupt.
    0x403fe392 in __select () at __select:-1
    -1    __select: No such file or directory.
        in __select
    (gdb) bt
    #0  0x403fe392 in __select () at __select:-1
    #1  0x400b8194 in __DTOR_END__ ()
       from /usr/local/gt4/lib/libglobus_xio_gcc32dbg.so.0
    #2  0x401532e6 in globus_callback_space_poll (timestop=0x804b6d0, space=-2)
        at globus_callback_nothreads.c:1424
    #3  0x40042de7 in globus_xio_read (user_handle=0x805d4a0, buffer=0x805cf98
"", 
        buffer_length=50, waitforbytes=50, nbytes=0xbfffdfe0, data_desc=0x0)
        at globus_xio_handle.c:3095
    #4  0x400244bb in globus_io_read (handle=0xbfffe27c, buf=0x805cf98 "", 
        max_nbytes=50, wait_for_nbytes=50, nbytes_read=0xbfffdfe0)
        at globus_io_xio_compat.c:3012
    #5  0x080493af in receiveCallback (cbArg=0xbfffe28c, ioHandle=0xbfffe27c, 
        gResult=0) at client.c:153
    #6  0x40026301 in globus_l_io_bounce_select_cb (arg=0xbfffe28c, 
        handle=0xbfffe27c, result=0, 
        buf=0x8049354 "...snip...", nbytes=0) at globus_io_xio_compat.c:4039
    #7  0x40020420 in globus_l_io_bounce_io_cb (xio_handle=0x805d4a0, result=0, 
        buffer=0x8049354 "...snip...", len=0, nbytes=0, data_desc=0x80750d8,
user_arg=0x805d168)
        at globus_io_xio_compat.c:798
    #8  0x4003c1ad in globus_l_xio_read_write_callback_kickout
(user_arg=0x80750d8)
        at globus_xio_handle.c:1148
    #9  0x4003c03f in globus_i_xio_read_write_callback (op=0x80750d8, result=0, 
        nbytes=0, user_arg=0x0) at globus_xio_handle.c:1116
    #10 0x40046a5c in globus_l_xio_driver_op_read_kickout (user_arg=0x80750d8)
        at globus_xio_driver.c:605
    #11 0x401532e6 in globus_callback_space_poll (timestop=0x804b6d0, space=-2)
        at globus_callback_nothreads.c:1424
    #12 0x40043be7 in globus_xio_write (user_handle=0x805d4a0, 
        buffer=0x805cf98 "", buffer_length=50, waitforbytes=50,
nbytes=0xbfffe284, 
        data_desc=0x0) at globus_xio_handle.c:3342
    #13 0x4002520b in globus_io_write (handle=0xbfffe27c, buf=0x805cf98 "", 
        nbytes=50, nbytes_written=0xbfffe284) at globus_io_xio_compat.c:3393
    #14 0x080492b6 in executeTest (argument=0xbfffe2c8) at client.c:102
    #15 0x0804917e in main (argc=5, argv=0xbfffe324) at client.c:48
    #16 0x403399af in __libc_start_main (main=0x8049150 <main>, argc=5, 
        ubp_av=0x0, init=0x8049f3c <__libc_csu_init>, 
        fini=0x8049f48 <__libc_csu_fini>, rtld_fini=0x40013020 <_rtld_global>, 
        stack_end=0x804cfb8) at ../sysdeps/generic/libc-start.c:152
    (gdb)
------- Comment #1 From 2006-04-18 01:39:30 -------
Created an attachment (id=926) [details]
myprogram.sh

[make]

    % sh myprogram.sh
    % make

[execute without encryption]

    % ./server 10001 100 & sleep 3 ; ./client localhost 10001 100
    [2] 21386
    [2]  - Done                          ./server 10001 100

[execute with encryption]

    % ./server --crypt=SSL 10002 100 & sleep 3; ./client --crypt=SSL localhost
10002 100
    [2] 21423
------- Comment #2 From 2006-04-18 02:14:19 -------
This problem is occurred with Non Thread flavor and is not occurred with
Pthread flavor.
------- Comment #3 From 2006-04-27 14:39:07 -------
As far as the client, you shouldn't use globus_io_read with a fixed read size,
since the select will fire the callback when any data is available, not
necessarily user available (decrypted) data.  You should use
globus_io_try_read, which can successfully return <nybtes when that is the
case.

For the server, you using the blocking globus_io_read and globus_io_write calls
in the accept callback is problematic when the app is built non-threaded.  The
result will be that each subsequent client connection will take control of the
process and earlier connections won't get returned to until the newer
connection is finished.  In general it is a bad idea to use blocking calls
within callbacks in our nonthreaded builds, but it is especially bad within a
server application.

Note that you may find it easier to deal with using
globus_io_register_read/write calls.  It is a little more work up front, but it
is easier to debug and IMO conceptually easier to follow.  Along those lines,
you may also want to use globus_xio.  It is the successor to globus_io, and
provides a common interface to a wide range of transports.  In the simple case
of a GSI tcp connection, it isn't much different, but it affords the ability to
easily change that to a clear tcp connection, clear udp connection, GSI udp
connection, etc.  See
http://www.globus.org/toolkit/docs/4.0/common/xio/developer-index.html for more
info on XIO.

Mike
------- Comment #4 From 2006-05-19 00:04:17 -------
> As far as the client, you shouldn't use globus_io_read with a fixed read size,
> since the select will fire the callback when any data is available, not
> necessarily user available (decrypted) data.  You should use
> globus_io_try_read, which can successfully return <nybtes when that is the
> case.

Thank you for your advice.

I modified my client to use globus_io_try_read(). However, it doesn't
work correctly. The outline of modification is shown below.

    receiveCallback()
    {
        globus_io_try_read();
        if (Was all data received? == Yes)
            It notify to main thread.
        else
            globus_io_register_select(receiveCallback);
    }

When this modification was performed, receiveCallback() came to be
called repeatedly. However, only 0 byte can always receive.

The example of execution of the client after modification is shown below.

    % ./client --crypt=SSL localhost 10000 100
    Receive 0bytes, Total 0bytes received.
    Receive 0bytes, Total 0bytes received.
    Receive 0bytes, Total 0bytes received.
    Receive 0bytes, Total 0bytes received.
    ....snip....
    Receive 0bytes, Total 0bytes received.
    Receive 0bytes, Total 0bytes received.
    Receive 0bytes, Total 0bytes received.
    Receive 0bytes,
    ---- Terminated by Ctrl-C ----
    %

The diff of modification is shown below.

    diff -u -r1.6 client.c
    --- client.c    1 Apr 2005 06:57:05 -0000       1.6
    +++ client.c    19 May 2006 04:57:26 -0000
    @@ -159,7 +159,7 @@

         /* Get the argument */
         monitor = cbArg;
    -    nBytes = monitor->nBytes / 2;
    +    nBytes = monitor->nBytes - monitor->receiveNbytes;

         /* Is error occurred? */
         if (gResult != GLOBUS_SUCCESS) {
    @@ -168,24 +168,30 @@
         }

         /* Receive the reply */
    -    gResult = globus_io_read(
    -       ioHandle, &monitor->buf[0], nBytes, nBytes, &nbytesRead);
    +    gResult = globus_io_try_read(
    +       ioHandle, &monitor->buf[monitor->receiveNbytes], nBytes,
&nbytesRead);
         if (gResult != GLOBUS_SUCCESS) {
            gtError(fName, gResult);
            abort();
         }
    -
    -    /* Receive the data */
    -    gResult = globus_io_read(
    -       ioHandle, &monitor->buf[nBytes], nBytes, nBytes, &nbytesRead);
    -    if (gResult != GLOBUS_SUCCESS) {
    -       gtError(fName, gResult);
    -       abort();
    +    monitor->receiveNbytes += nbytesRead;
    +    fprintf(stderr, "Receive %dbytes, Total %dbytes received.\n",
    +       nbytesRead, monitor->receiveNbytes);
    +
    +    /* Was all data received? */
    +    if (monitor->receiveNbytes >= monitor->nBytes) {
    +       /* Signal the monitor */
    +       monitorSignal(monitor);
    +    } else {
    +       /* Register the callback for receive */
    +       gResult = globus_io_register_select(
    +           ioHandle, receiveCallback, monitor, NULL, NULL, NULL, NULL);
    +       if (gResult != GLOBUS_SUCCESS) {
    +           gtError(fName, gResult);
    +           abort();
    +       }
         }

    -    /* Signal the monitor */
    -    monitorSignal(monitor);
    -
         /* Success */
         return;
     }
------- Comment #5 From 2006-05-19 00:05:52 -------
Created an attachment (id=958) [details]
Newer client.c

The client.c for Comment #4.
------- Comment #6 From 2006-05-19 00:16:31 -------
> For the server, you using the blocking globus_io_read and globus_io_write calls
> in the accept callback is problematic when the app is built non-threaded.  The
> result will be that each subsequent client connection will take control of the
> process and earlier connections won't get returned to until the newer
> connection is finished.  In general it is a bad idea to use blocking calls
> within callbacks in our nonthreaded builds, but it is especially bad within a
> server application.

Yes, probably, it will not be good to call blocking I/O in the accept
callback as surely you say. However, since the shown server program
shows the problem of a client, I think it is no problem.
------- Comment #7 From 2006-08-17 22:46:54 -------
I modified my program to specify only authentication. The deadlock
occurred like encryption with non thread flavor.

How to start the server and the client.

[server side]

    % ./server --crypt=authonly 10004 1024

[client side]

    % ./client --crypt=authonly ume.hpcc.jp 10004 1024

The diff of modification is shown below.

    % cvs diff -u
    Index: client.c
    ===================================================================
    RCS file: /grid/cvsroot/ngv2/experimental/bug251-GT3freeze/client.c,v
    retrieving revision 1.7
    diff -u -r1.7 client.c
    --- client.c    22 May 2006 07:24:07 -0000      1.7
    +++ client.c    18 Aug 2006 03:20:46 -0000
    @@ -268,13 +268,15 @@
         /* Check the arguments */
         if ((argc - i) != 3) {
     usage:
    -       fprintf(stderr, "Usage: %s [--crypt={GSI,SSL}] host port nBytes\n",
program);
    +       fprintf(stderr, "Usage: %s [--crypt={authonly,GSI,SSL}] host port
nBytes\n", program);
            exit(1);
         }

         /* Convert the crypt type */
         if (crypt != NULL) {
    -       if (strcmp(crypt, "GSI") == 0)
    +       if (strcmp(crypt, "authonly") == 0)
    +           argument->crypt = CRYPT_AUTHONLY;
    +       else if (strcmp(crypt, "GSI") == 0)
                argument->crypt = CRYPT_GSI;
            else if (strcmp(crypt, "SSL") == 0)
                argument->crypt = CRYPT_SSL;
    Index: server.c
    ===================================================================
    RCS file: /grid/cvsroot/ngv2/experimental/bug251-GT3freeze/server.c,v
    retrieving revision 1.5
    diff -u -r1.5 server.c
    --- server.c    13 Dec 2004 18:21:10 -0000      1.5
    +++ server.c    18 Aug 2006 03:20:46 -0000
    @@ -233,13 +233,15 @@
         /* Check the arguments */
         if ((argc - i) != 2) {
     usage:
    -       fprintf(stderr, "Usage: %s [--crypt={GSI,SSL} port nBytes\n",
program);
    +       fprintf(stderr, "Usage: %s [--crypt={authonly,GSI,SSL} port
nBytes\n", program);
            exit(1);
         }

         /* Convert the crypt type */
         if (crypt != NULL) {
    -       if (strcmp(crypt, "GSI") == 0)
    +       if (strcmp(crypt, "authonly") == 0)
    +           argument->crypt = CRYPT_AUTHONLY;
    +       else if (strcmp(crypt, "GSI") == 0)
                argument->crypt = CRYPT_GSI;
            else if (strcmp(crypt, "SSL") == 0)
                argument->crypt = CRYPT_SSL;
    Index: socket.c
    ===================================================================
    RCS file: /grid/cvsroot/ngv2/experimental/bug251-GT3freeze/socket.c,v
    retrieving revision 1.2
    diff -u -r1.2 socket.c
    --- socket.c    1 Apr 2005 06:57:05 -0000       1.2
    +++ socket.c    18 Aug 2006 03:20:46 -0000
    @@ -138,6 +138,8 @@
            gtError(fName, gResult);
            abort();
         }
    +    if (crypt == CRYPT_AUTHONLY)
    +       return;

         /* Initialize the authorization mode */
         *authMode = GLOBUS_IO_SECURE_AUTHORIZATION_MODE_SELF;
    Index: socket.h
    ===================================================================
    RCS file: /grid/cvsroot/ngv2/experimental/bug251-GT3freeze/socket.h,v
    retrieving revision 1.1
    diff -u -r1.1 socket.h
    --- socket.h    6 Dec 2004 09:44:10 -0000       1.1
    +++ socket.h    18 Aug 2006 03:20:46 -0000
    @@ -1,6 +1,7 @@
     /* Type of crypt */
     typedef enum cryptType_e {
         CRYPT_NONE,                /* Not crypted */
    +    CRYPT_AUTHONLY,    /* Authentication only */
         CRYPT_GSI,         /* Use GSI */
         CRYPT_SSL,         /* Use SSL */
     } cryptType_t;
------- Comment #8 From 2006-08-17 22:49:26 -------
Created an attachment (id=1038) [details]
My program, it was added authentication only.