/*
   todo
   build in control, for consistance of pairs filename / filenumber
   ( pioOpenFile member name, recv in tmpbuffer, if(!uniqueName(q,v,n))abort )
*/

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include <inttypes.h>
#include <stdio.h>
#include <stdbool.h>
#include <string.h>

#include <mpi.h>

#include "cdi.h"
#include "dmemory.h"
#include "namespace.h"
#include "pio.h"
#include "pio_comm.h"
#include "pio_impl.h"
#include "pio_util.h"

typedef struct
{
  struct dBuffer *db1;
  struct dBuffer *db2;
  struct dBuffer *db;
  FILE *fp;
  enum IO_Server_command command;
  int tsID, fileID;
  char name[];
} aFiledataPF;

static int
fileIDTestA(void *a, void *fileID)
{
  return ((aFiledataPF *)a)->fileID == (int)(intptr_t)fileID;
}

typedef struct
{
  long offset;
  bool finished;
  int fileID;
  bool nfinished[];
} bFiledataPF;

static int
fileIDTestB(void *a, void *fileID)
{
  return ((bFiledataPF *)a)->fileID == (int)(intptr_t)fileID;
}

static bool
fileIDCmpB(void *a, void *b)
{
  return ((bFiledataPF *)a)->fileID == ((bFiledataPF *)b)->fileID;
}

static listSet *bibAFiledataPF;

/***************************************************************/

static aFiledataPF *initAFiledataPF ( const char *key, size_t bs)
{
  aFiledataPF *afd;
  size_t len;
  int iret;

  len = strlen(key);
  afd = Calloc(1, sizeof (*afd) + len + 1);
  strcpy(afd->name, key);
  afd->tsID = 0;

  /* init output buffer */

  xdebug ( " name=%s, init output buffer",  afd->name );

  iret = dbuffer_init(&(afd->db1), bs);
  iret += dbuffer_init(&(afd->db2), bs);

  if ( iret > 0 )
    xabort("dbuffer_init did not succeed");

  afd->db = afd->db1;

  /* open file */
  xdebug ( "name=%s, open file",  afd->name );

  if ( ( afd->fp = fopen ( afd->name, "w" )) == NULL )
    xabort("Failed to open %s", afd->name);

  afd->command = IO_Open_file;
  return afd;
}

/***************************************************************/
static bFiledataPF *
initBFiledataPF(int fileID, int nc)
{
  bFiledataPF *bfd;
  size_t bfdSize = sizeof (bFiledataPF) + (size_t)nc * sizeof (bool);
  bfd = Calloc(1, bfdSize);
  bfd->offset = 0;
  bfd->finished = false;
  bfd->fileID = fileID;

  return bfd;
}

/***************************************************************/

static int
destroyAFiledataPF(void *v)
{
  int iret = 0;
  aFiledataPF *afd = ( aFiledataPF * ) v;

  /* close file */
  xdebug("name=%s, close file", afd->name);
  if ((iret = fclose(afd->fp)) == EOF)
    xabort("Failed to close %s", afd->name);

  /* file closed, cleanup */
  xdebug("name=%s, file closed, cleanup ...",  afd->name);
  dbuffer_cleanup(&(afd->db1));
  dbuffer_cleanup(&(afd->db2));

  Free(afd);

  return iret;
}

/***************************************************************/

static int
destroyBFiledataPF(void *v)
{
  int iret = 0;
  bFiledataPF *bfd = (bFiledataPF * ) v;

  Free ( bfd );

  return iret;
}

/***************************************************************/

static bool
compareNamesAPF(void *v1, void *v2)
{
  aFiledataPF *afd1 = v1, *afd2 = v2;

  return !strcmp(afd1->name, afd2->name);
}

/***************************************************************/

static void
fpgPOSIXFPGUARDSENDRECV(void)
{
  int i, source, iret;
  struct fileOpTag rtag;
  MPI_Status status;
  bFiledataPF *bfd;
  listSet *bibBFiledataPF;
  long amount;
  MPI_Comm commPio = commInqCommPio();
  int nProcsColl =  commInqSizeColl();
  bool * sentFinalize, doFinalize = false;

  xdebug ( "ncollectors=%d", nProcsColl );

  bibBFiledataPF = listSetNew( destroyBFiledataPF, fileIDCmpB);
  sentFinalize = Calloc((size_t)nProcsColl, sizeof (sentFinalize[0]));

  for ( ;; )
    {
      xmpi ( MPI_Probe ( MPI_ANY_SOURCE, MPI_ANY_TAG, commPio, &status ));
      source = status.MPI_SOURCE;
      rtag = decodeFileOpTag(status.MPI_TAG);

      xdebug("receive message from source=%d, id=%d, command=%d ( %s )",
             source, rtag.id, rtag.command, cdiPioCmdStrTab[rtag.command]);

      switch (rtag.command)
      	{
      	case IO_Open_file:

          if (!(bfd = listSetGet(bibBFiledataPF, fileIDTestB,
                                 (void *)(intptr_t)rtag.id)))
	    {
	      bfd = initBFiledataPF(rtag.id, nProcsColl);

	      if ((iret = listSetAdd(bibBFiledataPF, bfd)) < 0)
		xabort("fileID=%d not unique", rtag.id);
              bfd->fileID = iret;
	    }

          xdebug("id=%d, command=%d ( %s ), send offset=%ld", rtag.id,
rtag.command, cdiPioCmdStrTab[rtag.command], bfd->offset);

	  xmpi ( MPI_Sendrecv ( &( bfd->offset ), 1, MPI_LONG, source,  status.MPI_TAG,
                                &amount, 1, MPI_LONG, source,  status.MPI_TAG,
                                commPio, &status ));

	  bfd->offset += amount;

          xdebug("id=%d, command=%d ( %s ), recv amount=%ld, set offset=%ld",
                 rtag.id, rtag.command, cdiPioCmdStrTab[rtag.command], amount,
                 bfd->offset);

	  break;

	case IO_Set_fp:

          if (!(bfd = listSetGet(bibBFiledataPF, fileIDTestB,
                                 (void *)(intptr_t)rtag.id)))
            xabort("fileId=%d not in set", rtag.id);

          xdebug("id=%d, command=%d ( %s ), send offset=%ld", rtag.id,
                 rtag.command, cdiPioCmdStrTab[rtag.command], bfd->offset);

	  xmpi ( MPI_Sendrecv ( &( bfd->offset ), 1, MPI_LONG, source,  status.MPI_TAG,
                                &amount, 1, MPI_LONG, source,  status.MPI_TAG,
                                commPio, &status ));

	  bfd->offset += amount;

          xdebug("id=%d, command=%d ( %s ), recv amount=%ld, set offset=%ld",
                 rtag.id, rtag.command, cdiPioCmdStrTab[rtag.command], amount,
                 bfd->offset);

	  break;

	case IO_Close_file:

          if (!(bfd = listSetGet(bibBFiledataPF, fileIDTestB,
                               (void *)(intptr_t)rtag.id)))
            xabort("fileId=%d not in set", rtag.id);

          xdebug("id=%d, command=%d ( %s )), send offset=%ld", rtag.id,
                 rtag.command, cdiPioCmdStrTab[rtag.command], bfd->offset);

	  xmpi ( MPI_Sendrecv ( &( bfd->offset ), 1, MPI_LONG, source,  status.MPI_TAG,
                                &amount, 1, MPI_LONG, source,  status.MPI_TAG,
                                commPio, &status ));

	  bfd->offset += amount;

          xdebug("id=%d, command=%d ( %s ), recv amount=%ld, set offset=%ld",
                 rtag.id, rtag.command, cdiPioCmdStrTab[rtag.command], amount,
                 bfd->offset);


	  bfd->nfinished[source] = true;
	  bfd->finished          = true;

	  for ( i = 0; i < nProcsColl; i++ )
	    if ( !( bfd->nfinished[i] ))
	      {
		bfd->finished = false;
		break;
	      }

	  if ( bfd->finished )
            listSetRemove(bibBFiledataPF, fileIDTestB,
                          (void *)(intptr_t)rtag.id);
          break;
        case IO_Finalize:
          {
            int buffer = CDI_UNDEFID, collID;

            xmpi ( MPI_Recv ( &buffer, 1, MPI_INT, source, status.MPI_TAG,
                              commPio, &status ));
            sentFinalize[source] = true;
            doFinalize = true;
            for ( collID = 0; collID < nProcsColl; collID++ )
              doFinalize &= sentFinalize[collID];
            if ( doFinalize )
              {
                if (!listSetIsEmpty(bibBFiledataPF))
                  xabort("set bibBFiledataM not empty");
                else
                  {
                    xdebug("%s", "destroy set");
                    listSetDelete(bibBFiledataPF);
                  }
                Free(sentFinalize);
                return;
              }
          }
          break;
        default:
          xabort ( "COMMAND NOT IMPLEMENTED" );
	}
    }
}

//*******************************************************

static void
writePF(aFiledataPF *afd)
{
  size_t amount, written;
  long offset;
  long amountL;
  int error, tag;
  MPI_Status status;
  /* FIXME: pretend there's only one special rank for now */
  int specialRank = commInqSizePio() - 1;
  MPI_Comm commPio = commInqCommPio();

  /* send buffersize, recv offset */

  amount = dbuffer_data_size ( afd->db );
  amountL = ( long ) amount;
  int id = afd->fileID;
  tag = encodeFileOpTag(id, afd->command);

  xmpi ( MPI_Sendrecv ( &amountL, 1, MPI_LONG, specialRank, tag,
                        &offset, 1, MPI_LONG, specialRank, tag,
                        commPio, &status ));
  xdebug ( "id=%d, command=%d, amount=%zu, send amountL=%ld, recv offset=%ld",
           id, afd->command, amount, amountL, offset );

  /* write buffer */

  if (( error = fseek ( afd->fp, offset, SEEK_SET )) != 0 )
    xabort ( "did not succeed seeking fp" );

  if (( written =
	fwrite ( afd->db->buffer, sizeof ( char ), amount, afd->fp )) !=
      amount )
    xabort("fileId=%d, expect to write %zu byte, written %zu byte",
           id, amount, written);

  xdebug("written %zu bytes in file %d with offset %ld",
         written, id, offset);

  /* change outputBuffer */

  dbuffer_reset ( afd->db );

  if ( afd->db == afd->db1 )
    {
      xdebug ( "id=%d, change to buffer 2 ...", id );
      afd->db =  afd->db2;
    }
  else
    {
      xdebug ( "id=%d, change to buffer 1 ...", id );
      afd->db =  afd->db1;
    }

  afd->command = IO_Set_fp;
}


/***************************************************************/

static void
defTimestepPF(aFiledataPF *afd, int tsID)
{
  if ( afd == NULL || tsID < 0 || tsID != afd->tsID + 1 )
    xabort ( " defTimestepPF() didn't succeed." );
  afd->tsID = tsID;
}


/***************************************************************/

static void
flushOp(aFiledataPF *a, int tsID)
{
  writePF(a);
  defTimestepPF(a, tsID);
}


static size_t
fwPOSIXFPGUARDSENDRECV(int fileID, const void *buffer, size_t len, int tsID)
{
  int error = 0;
  int filled = 0;
  aFiledataPF *afd
    = listSetGet(bibAFiledataPF, fileIDTestA, (void *)(intptr_t)fileID);

  bool flush = tsID != afd->tsID;

  if (flush)
    {
      xdebug("fileID %d, tsID = %d, flush buffer", fileID, tsID);
      flushOp(afd, tsID);
      xmpi ( MPI_Barrier ( commInqCommColl ()));
    }

  filled = dbuffer_push(afd->db, ( unsigned char * ) buffer, len);

  xdebug ( "fileID = %d, tsID = %d, pushed data on buffer, filled = %d",
           fileID, tsID, filled );

  if ( filled == 1 )
    {
      if ( flush )
	error = filled;
      else
	{
	  writePF(afd);

	  error = dbuffer_push ( afd->db, ( unsigned char * ) buffer, len );
	}
    }

  if ( error == 1 )
    xabort("did not succeed filling output buffer, fileID=%d", fileID);

  return len;
}

/***************************************************************/

static int
fcPOSIXFPGUARDSENDRECV(int id)
{
  aFiledataPF *afd;
  int iret;

  xdebug("write buffer, close file %d and cleanup", id);

  afd = listSetGet(bibAFiledataPF, fileIDTestA, (void *)(intptr_t)id);

  afd->command = IO_Close_file;

  writePF(afd);

  /* remove file element */
  iret = listSetRemove(bibAFiledataPF, fileIDTestA, (void *)(intptr_t)id);
  /* make sure the file is closed on all collectors before proceeding */
  xmpi(MPI_Barrier(commInqCommColl()));
  return iret;
}

/***************************************************************/
static void
elemCheck(void *q, void *nm)
{
  aFiledataPF *afd = q;
  const char *name = nm;

  if (!strcmp(name, afd->name))
    xabort("Filename %s has already been added to set\n", name);
}

static int
fowPOSIXFPGUARDSENDRECV(const char *filename, const char *mode)
{
  int id;
  enum {
    bcastRoot = 0
  };
  if ((mode[0] != 'w' && mode[0] != 'W') || mode[0] == 0 || mode[1] != 0)
    xabort("Unsupported mode \"%s\" in parallel file open.", mode);
  aFiledataPF *afd;
  static unsigned long buffersize = 0;

  /* broadcast buffersize to collectors */
  if (!buffersize)
    {
      if (commInqRankColl() == bcastRoot)
        buffersize = findWriteAccumBufsize();
      xmpi(MPI_Bcast(&buffersize, 1, MPI_UNSIGNED_LONG, bcastRoot,
                     commInqCommColl()));
    }

  /* init and add file element */
  listSetForeach(bibAFiledataPF, elemCheck, (void *)filename);

  afd = initAFiledataPF(filename, (size_t)buffersize);

  if ((id = listSetAdd(bibAFiledataPF, afd)) < 0)
    xabort("filename %s not unique", afd->name);
  afd->fileID = id;
  xdebug("name=%s, init and add aFiledataPF, return id = %d",
         filename, id);
  {
    long offset, amount = 0L;
    int tag = encodeFileOpTag(afd->fileID, afd->command);
    int specialRank = commInqSpecialRank ();
    MPI_Status status;
    MPI_Comm commPio = commInqCommPio ();
    xmpi(MPI_Sendrecv(&amount, 1, MPI_LONG, specialRank, tag,
                      &offset, 1, MPI_LONG, specialRank, tag,
                      commPio, &status));
  }
  afd->command = IO_Set_fp;
  return id;
}

/***************************************************************/

static void
finalizePOSIXFPGUARDSENDRECV(void)
{
  int buffer = 0, tag = encodeFileOpTag(0, IO_Finalize);

  xmpi(MPI_Send(&buffer, 1, MPI_INT, commInqSpecialRank(),
                tag, commInqCommPio()));

  if (!listSetIsEmpty(bibAFiledataPF))
    xabort("set bibAFiledataM not empty");
  else
    {
      xdebug("%s", "destroy set");
      listSetDelete(bibAFiledataPF);
    }
}

/***************************************************************/

void
initPOSIXFPGUARDSENDRECV(void)
{
  int numIOServers = commInqSizePio();
  if (numIOServers < 2)
    xabort("error: # of I/O processes must be >= 2 for mode, but is %d",
           numIOServers);

  int isCollector = commInqRankColl() != -1;
  if (!isCollector)
    fpgPOSIXFPGUARDSENDRECV ();
  else
    {
      namespaceSwitchSet(NSSWITCH_FILE_OPEN,
                         NSSW_FUNC(fowPOSIXFPGUARDSENDRECV));
      namespaceSwitchSet(NSSWITCH_FILE_CLOSE,
                         NSSW_FUNC(fcPOSIXFPGUARDSENDRECV));
      namespaceSwitchSet(NSSWITCH_FILE_WRITE,
                         NSSW_FUNC(fwPOSIXFPGUARDSENDRECV));
      cdiPioFileWritingFinalize = finalizePOSIXFPGUARDSENDRECV;
      bibAFiledataPF = listSetNew(destroyAFiledataPF, compareNamesAPF);
    }
}

/*
 * Local Variables:
 * c-file-style: "Java"
 * c-basic-offset: 2
 * indent-tabs-mode: nil
 * show-trailing-whitespace: t
 * require-trailing-newline: t
 * End:
 */
