llcontrol-core.cpp

Go to the documentation of this file.

/*****************************************************************************
 *
 * File: llcontrol-core.cpp
 *
 * $RCSfile: llcontrol-core.c,v $
 * 
 * Copyright (C) 2001 D-TACQ Solutions Ltd
 * not to be used without owner's permission
 *
 * Description:
 *     application implementing the LOW LATENCY CONTROL feature
 *
 */


/** @file llcontrol-core.cpp module implements the core of llcontrol loop.
 *
 - mmapMboxes() - get app mapping for mailboxes

 - mmapBuffer() - get dmabuff and phys addresses

 - runSCM() - capture using soft trigger

 - runECM() - capture using external clock
<p>
<h2>Linux 2.4 Host Only:</h2>
2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4

  Note on HOST MEMORY MAP: assumes bootcommand configured as follows,
  to allow a 16MB host area per card in at the top of memory.
  This host area will be slaved off the pci backplane

 - mem=NNN restrict memory for Linux. Must allow Nx16MB + gap
 - acq32_big_buf_base=0xnnnn : informs driver of start of area
 - acq32_big_buf_len=0xxxxx  : informs driver of length of area

eg

[dt100@cp605 dt100]$ cat /proc/cmdline mem=320M acq32.load_order=slotbased 

 acq32_big_buf_base=0x18000000 acq32_big_buf_len=0x08000000

For the purpose of LLCONTROL, each 16MB card are is used as follows:

- 0x000000 .. 0xefffff (15MB) - AI reception area (controlled by client app).
- 0xf00000 .. 0xffffff (1MB)  - message buffers (owned by target).

2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4 2.4


This means that if the llcontrol app is using an incrementing memory
strategy in host buffer for AI, this is subject to max 15MB
(80K samples at 96 channels).

The message buffers are used for DAC output. 
The application reserves a Mesage Frame Address (MFA) from the target 
before copying data to slave memory.
The MFA is an index into the message buffer memory.

</p>

<h2>Linux 2.6 Host: </h2>
the host buffer is setup dynamically, but is LIMITED to 4MB, 
of which the top 1M is used for messaging.

*/



#include "local.h"

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>

#include <errno.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <stdio.h>
#include <unistd.h>

#include <popt.h>

#include "acq32ioctl.h"
#include "acq32busprot.h"

#include "llif.h"
#include "llprotocol.h"


#include "llcontrol.h"
#include "x86timer.h"

#define FLAVOR "ACQ196"

#include "llcontrol-core.h"

const char* core_ident = "$Revision: 1.1.4.33 $ B1102\n";

 int G_quit;



unsigned  llcv2_hb_offset = 0;





void v2_updateTstats(
        u32 cmd, struct Card* card, struct TimingStats* tstats)
/** updates timing stats from embedded host buffer data */
{
        u32* stats = getVaddr(card->buf, LLCV2_OFFSET_STATUS_HSBT);
        tstats->tinst = llv2_extend32(stats[BP_MB_LLC_TINST], 
                                      stats[LLCV2_STATUS_TINST]);
        tstats->tprocess = LLC_GET_TCYCLE(stats[BP_MB_LLC_CSR]);
}


void (*updateTstats)(u32 cmd, struct Card* card, struct TimingStats* tstats) = 
                v2_updateTstats;


u32 card_v2_WaitDmaDone(struct Card* card)
{
        return llv2WaitDmaDone(card->mbx, 
                               getVaddr(card->buf, LLCV2_OFFSET_STATUS_HSBT));
}



u32 (*waitDmaDone)(struct Card *c) = card_v2_WaitDmaDone;
/**< virtual function to block until DMA done.
 *  defaults to V1 mbox
 */

        
int user_abort = 0;    /** @todo could be set by a signal */



static int measureBridge(struct MU *mu) 
/** Run a test - measureBridge performance. */
{
#define NMEASURE 20
        static struct {
                unsigned elapsed;
                u32 mbox;
        } stats[NMEASURE];
        int im;
        unsigned sum = 0;
        
        for (im = 0; im != NMEASURE; ++im){
                INIT_TIMER;
                stats[im].mbox = getMbox(mu, 3);
                stats[im].elapsed = get_elapsed_microseconds(0);

                usleep(10000);
        }      

        fprintf(stderr, "[%2s], %10s, %10s\n","id", "mbox3", "usecs");
        for (im = 0; im != NMEASURE; ++im){
                sum += stats[im].elapsed;

                fprintf(stderr, "[%2d], 0x%08x, %10d\n",
                        im, stats[im].mbox, stats[im].elapsed);
        }

        fprintf(stderr, "mean  %10s, %12.1f\n", "", (float)sum/NMEASURE);
        return 0;
}

static int measureBridgeStats(struct TestDescription *td, struct MU *mu)
{
        static struct {
                int min;
                int max;
                int sum;
                int samples;
        } stats = { 1000000, -1000000, 0, 0 };

        int im;
        int current;

        if ( td->mask_ints ){
                PRINTF(1)( "mask ints %s\n", td->mask_ints_mask );
                FOREACHCARD{
                        acq200_setImask(EACHSLOT(td), td->mask_ints_mask);
                }
        }

        for (im = 0; im != td->iterations; ++im){
                INIT_TIMER;
                getMbox(mu, 3);
                
                current = get_elapsed_microseconds(0);

                if (current < stats.min){
                        stats.min = current;
                }else if (current > stats.max){
                        stats.max = current;
                }
                stats.sum += current;
                stats.samples++;
        }

        if (td->mask_ints){             // @@todo refactor me please.
                char* enable_mask = new char[strlen(td->mask_ints_mask)+1];
                int ic;

                for (ic = 0; td->mask_ints_mask[ic]; ++ic){
                        enable_mask[ic] = '0';                          /** enable all! */
                }
                enable_mask[ic] = '\0';
                FOREACHCARD{
                        acq200_setImask(EACHSLOT(td), enable_mask);
                }
                delete [] enable_mask;
        }

        fprintf(stderr, "measureBridgeStats results from %d polls\n", 
                stats.samples);
        fprintf(stderr, "%10s %d\n", "min", stats.min);
        fprintf(stderr, "%10s %d\n", "max", stats.max);
        fprintf(stderr, "%10s %.2f\n", "mean", (float)stats.sum/stats.samples);
        return 0;
}



#include <signal.h>

static struct TestDescription* S_td;

static void quit_handler( int signum )
{
        struct TestDescription* td = S_td;

        fprintf( stderr, "Quitting Time - mailboxes were\n" );

        FOREACHCARD{
                fprintf(stderr, "0x%08x  0x%08x  0x%08x  0x%08x\n",
                        getMbox(EACHMBX(td), 0), 
                        getMbox(EACHMBX(td), 1),    
                        getMbox(EACHMBX(td), 2),
                        getMbox(EACHMBX(td), 3) );
        }

        FOREACHCARD{
                showLastWrites(EACHMBX(S_td));
        }

        FOREACHCARD{
                leaveLLC(EACHMBX(S_td));
        }
        exit( signum );
}

void setupAbortHandler( struct TestDescription* td )
{
        static struct sigaction  def_action = { { quit_handler } };

        S_td = td;    // ugly global. use class var in C++
    
        sigaction( SIGINT, &def_action, 0 );
}




static void monitor_handler( int signum )
{
        static u32 old_mboxes[4];
        u32 new_mboxes[4];
        int ibox;
        int changed = 0;

        for ( ibox = 0; ibox != 4; ++ibox ){
                new_mboxes[ibox] = getMbox( S_td->cards[0].mbx, ibox );
                if ( new_mboxes[ibox] != old_mboxes[ibox] ){
                        changed = 1;
                }
        }
        
        if ( changed || verbose > 2 ){
                fprintf( stderr, "%8d ", S_td->iter );
                for ( ibox = 0; ibox != 4; ++ibox ){
                        fprintf( stderr, "0x%08x%c ", 
                                 new_mboxes[ibox], 
                                 new_mboxes[ibox]!=old_mboxes[ibox]? '*': ' ' );
                            
                        old_mboxes[ibox] = new_mboxes[ibox];
                }
                fprintf( stderr, "\n" );
        }
        if ( G_quit ){
                setupMonitor( 0 );
                fprintf( stderr, "monitor quitting\n" );
        }           
}

void setupMonitor( int millisec )
{
        static struct sigaction monitor_action = { { monitor_handler } };
        static struct itimerval new_timer = {};

        new_timer.it_value.tv_usec    =
                new_timer.it_interval.tv_usec = millisec*1000;
    
        sigaction( SIGALRM, &monitor_action, 0 );
        setitimer( ITIMER_REAL, &new_timer, 0 );
}

void initCardResource(struct Card* card)
{
        card->mbx = mmapMbox(card->slot);
        card->buf = mmapBigBuffer(card->slot, ACQ196_BIGBUF_AREA);
        hbPrimeBuffer(card->buf);

        get_cpu_clock_speed();
}


static void debug_prompt(int icard, int ibuf, u32 addr)
/** promnpt user to ensure pram set (temporary pre-update measure). */
{
        const char* value;
        switch(ibuf){
        case LLCV2_INIT_AI_HSBT:
                value = "AI_target";    break;
        case LLCV2_INIT_AO_HSBS:
                value = "AO_src";               break;
        case LLCV2_INIT_DO_HSBS:
                value = "DO src";               break;
        case LLCV2_INIT_STATUS_HSBT:
                value = "STATUS_target";break;
        default:
                value = "";
        }

        fprintf(stderr,
                "V2: please ensure card %d has pram %s set 0x%08x\n",
                icard, value, addr);
}


static void sync_2v_updateTstats(
        u32 cmd, struct Card* card, struct TimingStats* tstats)
/** updates timing stats from embedded host buffer data */
{
#define SAMPLE_SIZE (96*2)   /* WORKTODO */
        u32* stats = getVaddr(card->buf, card->sync_2v_offset_status_hsbt);
        tstats->tinst = llv2_extend32(stats[LLC_SYNC2V_IN_MBOX2],
                                      stats[LLC_SYNC2V_IN_TINST]);
        tstats->tprocess = LLC_GET_TCYCLE(stats[LLC_SYNC2V_IN_MBOX0]);
}

static u32 card_sync_2v_WaitDmaDone(struct Card* card)
{
        return card->tlatch = llv2WaitDmaDone_2v(card->mbx,
                  getVaddr(card->buf, card->sync_2v_offset_status_hsbt),
                  card->tlatch
        );
}

#define BAR_FIFO        3


static u32 getSlavePa(int slot)
{
        char fname[128];
        char line[80];
        FILE *fp;
        int bar;
        u32 pa = 0;

        sprintf(fname, "/dev/ao32cpci/ctrl/ao32cpci.%d/resource", slot);

        fp = fopen(fname, "r");
        if (!fp){
                fprintf(stderr, "ERROR: failed to open \"%s\"\n", fname);
                exit(errno);
        }

        for (bar = 0; fgets(line, 80, fp); ++bar){
                if (bar == BAR_FIFO){
                        long long pa0, pa1, len;
                        if (sscanf(line,
                                "%Lx %Lx %Lx", &pa0, &pa1, &len) == 3){
                                pa = (u32)pa0;
                        }
                }
        }
        fclose(fp);

        if (pa == 0){
                fprintf(stderr,
                        "ERROR: failed to get sensible PA for %d\n", slot);
        }
        return pa;
}


static void setSlaveData(u32* aovec, void* src)
/** create some data for AO32CPCI.
  AO32 gets AO16 data duplicated, DO64 gets a walking bit
*/
{
        static int ibit;
        unsigned long long DO_PAT = 1ULL<<ibit;

        if (++ibit > 64){
                ibit = 0;
        }

        memcpy(aovec, src, 16*sizeof(short));
        memcpy(aovec+8, src, 16*sizeof(short));
        memcpy(aovec+16, &DO_PAT, sizeof(unsigned long long));
}




void appEnterLLC_SYNC_2VAO32(
        int icard, struct MU *mu, struct TestDescription *td)
/** set up LLCV2_INIT buffer and enter mode.
 *  Buffer set up as 4K block at offset 0
*/
{
        u32* init_buf = getVaddr(EACHBUF(td), 0);
        u32 target_pa = getBusAddr(EACHBUF(td), 0);
        struct Card* card = &td->cards[icard];
        int islave = 0;
        PRINTF(2)("appEnterLLC_SYNC_2V() va:%p pa:0x%08x %s slaves %d\n",
                  init_buf, target_pa, MASTER? "MASTER": "",td->ao32_count);


        /** set up for single 4K buffer */
        llcv2_hb_offset = 0;
        card->sync_2v_offset_status_hsbt =
                LLCV2_AI_HSBT + card->channels*sizeof(short);
        card->tlatch = 0;

        /** uses V2 synchronization */
        updateTstats = sync_2v_updateTstats;
        waitDmaDone  = card_sync_2v_WaitDmaDone;

        init_buf[LLCV2_INIT_MARKER] = LLCV2_INIT_MAGIC_AO32;
        init_buf[LLCV2_INIT_AI_HSBT] = target_pa + LLCV2_AI_HSBT;
        init_buf[LLCV2_INIT_AO_HSBS] = target_pa + LLCV2_AO_HSBS;

/* make a zero terminated list of slave pa's */
        if (MASTER){
                for (islave = 0; islave < td->ao32_count; ++islave){
                        init_buf[LLCV2_INIT_AO32PA0+islave] =
                                getSlavePa(td->ao32_ids[islave]);
                }
        }
        init_buf[LLCV2_INIT_AO32PA0+islave] = 0;

        enterLLC_SYNC_ECM(
                mu, td->clkpos, td->trpos,
                td->arg.divisor,td->internal_loopback,
                BP_FC_SET_LLCV2_INIT,
                target_pa );
}


static int make_output_file(
        struct TestDescription* td,
        int slot,
        const char* _class)
{
        if (td->outfname == 0){
                return 0;                // no file
        }else if ( td->outfname[0] == '-' ){
                return 1;                // use stdout
        }else{
                char outfname[128];
                int fd_out;

                sprintf(outfname, "%s.%d.%s", td->outfname, slot, _class);
                fd_out = open( outfname, O_WRONLY|O_CREAT|O_TRUNC, 0666 );

                if (fd_out <= 0){
                        fprintf(stderr, "ERROR: failed to open file %s\n",
                                outfname);
                        exit( -1 );
                }
                return fd_out;
        }
}
static void doWorkBufDataOutput(struct TestDescription *td)
{
        FOREACHCARD{
                int iter;
                int offset = 0;
                int fd_out = make_output_file(td, EACHSLOT(td), "userbuf");

                for (iter = 0; iter != td->iterations; ++iter){
                        write(fd_out, (char*)td->work_buf[icard]+offset,
                              td_sample_size(td)*td->samples);
                        offset += td_sample_size(td)*td->samples;
                }
                close(fd_out);
        }
}
static void doDmaBufDataOutput(struct TestDescription *td)
{
        FOREACHCARD{
                int iter;
                int offset = 0;
                int fd_out = make_output_file(td, EACHSLOT(td), "dmabuf");

                for (iter = 0; iter != td->iterations; ++iter){
                        write(fd_out, getVaddr(EACHBUF(td), offset),
                              td_sample_size(td)*td->samples);
                        if (td->overwrite){
                                break;
                        }
                        offset += td->sample_offset;

                }
                close(fd_out);
        }
}
static int CLIP_LIMIT = 100000;

/** CLIP bogus times - not to try hide anything - big number spoils display */
#define CLIP(t) ((t) < 0? -1: (t) > CLIP_LIMIT? (-1): (t))

static inline int extractTprocess(struct TimingStats *ts)
/** if TP is available, USE IT, else calculate from tlatch, tinst
 *  this is valid because either it is accurate IOP calc, or
 *  worst case Host calc
 */
{
                return CLIP(ts->tinst) - CLIP(ts->tlatch);
}

static void dumpTimingStats( struct TestDescription* td, FILE* fp )
{
#define HFMT "%8s, %10s, %10s, %8s, %8s, %6s, %4s, "
#define DFMT "%8d, %10d, %10d, %8d, %8d, %6d, %4d, "

        int iter;

        u32 old_tlatch[MAXCARDS] = {};

        FOREACHCARD{
                fprintf(fp,  "%60s: %d", "CARD", EACHSLOT(td));
        }
        fprintf(fp, "\n");
        FOREACHCARD{
                int it;
                fprintf(fp,  HFMT,
                       "iter", "tinst", "tlatch",  "tprocess", "tclock",
                       "hpol","tpol");

                if (td->tlog < 2) continue;                     /** brief report */

                for (it = 0; it != MAXTEST; ++it){
                        fprintf(fp,  "%6d,", it);
                }
        }
        fprintf(fp, "\n");

        for (iter = 0; iter != td->iterations; ++iter){

                if ( iter && td->stats_buf[0][iter].iter == 0 ){
                        fprintf(fp, "\n");
                        break;
                }else

                FOREACHCARD{
                        struct TimingStats *ts = &td->stats_buf[icard][iter];
                        int tclock = ts->tlatch - old_tlatch[icard];
                        int it;
                        int tp1 = 0;
                        int tp2 = 0;

                        fprintf(fp,  DFMT,
                                ts->iter,
                                CLIP(ts->tinst),
                                CLIP(ts->tlatch),
                                extractTprocess(ts),
                                CLIP(tclock),
                                ts->hb_poll,
                                ts->target_poll);
                        old_tlatch[icard] = ts->tlatch;

                        if (td->tlog < 2) continue;                     /** brief report */

                        for (it = 0; it != MAXTEST; ++it){
                                int tp = ts->test_points[it];
                                if (tp){
                                        if (tp1 == 0){
                                                tp1 = tp;
                                        }else if (tp > tp2){
                                                tp2 = tp;
                                        }
                                }
                                fprintf(fp,  "%6d,", tp);
                        }
                        fprintf(fp, " T:%3d, ", tp2 - tp1);
                }

                fprintf(fp, "\n");
        }
#undef HFMT
#undef DFMT
#undef CLIP
}


static void dumpTimingStatsBin(struct TestDescription* td, const char *bfile)
{
        FILE* fp = fopen(bfile, "w");

        u32 tclock;
        int iter;
        u32 old_tinst[MAXCARDS] = {};

        assert(fp);

        for ( iter = 0; iter != td->iterations; ++iter ){
                FOREACHCARD{
                        struct TimingStats *ts = &td->stats_buf[icard][iter];

                        fwrite(&ts, sizeof(struct TimingStats), 1, fp);
                        tclock = ts->tinst - old_tinst[icard];
                        fwrite(&tclock, sizeof(u32), 1, fp);
                        old_tinst[icard] = ts->tinst;
                }
        }
        fclose(fp);
}

void doPostShotAnalysis(struct TestDescription *td)
{
        if ( td->tlog ){
                if (td->tlog_binfile != 0){
                        dumpTimingStatsBin(td, td->tlog_binfile);
                }else{
                        FILE *fp = fopen("/tmp/llcontrol.tstats", "w");
                        dumpTimingStats(td, fp);
                        fclose(fp);
                        printf("timing stats stored in /tmp/llcontrol.tstats\n");
//                      dumpTimingStats(td, stdout);
                }
        }
        if ( td->outfname != 0 ){
                doDmaBufDataOutput(td);
                if (td->do_work){
                        doWorkBufDataOutput(td);
                }
        }
}




void updateTimingStats(
        struct TimingStats* buffer, int iter, struct TimingStats* tstats)
{
        struct TimingStats* cursor = &buffer[iter];

        memcpy(cursor, tstats, sizeof(struct TimingStats));
        cursor->iter = iter;
}