SPE to SPE DMA transfer

From Cellbe

This article shows a simple DMA transfer between two SPEs. The synchronization is done via mailbox with the aid of the PPE. The process "PGET" holds the target memory area; it writes 200 in the target area, then waits for the transfer, and prints the value stored in target area. The process "PPUT" writes a value in the target memory area. The order of operations is as follows for custom research papers:

1. The SPE processes are started
2. PGET communicates to the PPE the target address via mailbox, then synchronizes with the PPE with a mailbox read
3. The PPE calculates the effective address of the target, and sends it to PPUT via mailbox, and synchronizes with PPUT with a mailbox read
4. PPUT reads the target's effective address from mailbox, starts the dma trasfer, waits for it to finish and wakes up the PPE with a mailbox write
5. The PPE wakes up, and writes a mailbox to PGET to wake it up.
6. PGET wakes up, and prints the content of the target memory area.

Contents 1 Considerations 2 Code snippet: Pointer passing structure 3 Code snippet: PPE side 4 Code snippet: process PGET, SPE side 5 Code snippet: process PPUT, SPE side

Considerations

Please note that this kind of synchronization is not recommended, because it involves the PPE. Signals would be surely better. Also the PPE waits actively for the mailbox messages. while it should do some kind of passive wait. Synchronization through mailbox has been chosen just because of simplicity.

Code snippet: Pointer passing structure

This structure is used to ensure compatibility when compiling with 32 or 64 bits word size.

/* =============================================================================
 *       Filename:  common.h
 *    Description:  common data structure used to pass pointers
 *         Author:  Alessandro Piras
 * =============================================================================
 */
#pragma once
union eat{
    struct{
        unsigned h;
        unsigned l;
    };  
    unsigned long long e;
    struct{
#ifndef __powerpc64__
        unsigned int reserved;
#endif  
        volatile void *p;
    };  
};  

Code snippet: PPE side

/* =============================================================================
 *       Filename:  dmaspe.c							
 *    Description:  Starts the spu threads and cares of synchronizing them with 
 *		    Mailboxes							
 *         Author:  Alessandro Piras (laynor@gmail.com)				
 *        Company:								
 * ===========================================================================*/


#include <libspe2.h>
#include <stdio.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>
#include "common.h"

#define MYALIGN(vardecl, n) vardecl\\
	__attribute__ ((__aligned__ (n)))

extern spe_program_handle_t dmaspespu_get;
extern spe_program_handle_t dmaspespu_put;

MYALIGN(unsigned int ls_addr, 128);

/************************************
 * Routine used to start the threads*
 ***********************************/

void *pthrtn(void *arg)
{
	unsigned int entry = SPE_DEFAULT_ENTRY;
	spe_context_ptr_t ctx;

	ctx = *((spe_context_ptr_t *) arg);
	spe_context_run(ctx, &entry, 0, &ls_addr, NULL, NULL);
	pthread_exit(NULL);
}

int main()
{
	spe_context_ptr_t ctx1, ctx2;
	union eat shared_lsaddr;
	pthread_t pth1, pth2;
	union eat ls_base_addr;
	union eat shared_ea;
	unsigned int bell;
	
	/* Creating the contexts, loading the programs and starting threads   */
	ctx1 = spe_context_create(0,NULL);
	ctx2 = spe_context_create(0,NULL);
	
	spe_program_load(ctx1, &dmaspespu_get);
	spe_program_load(ctx2, &dmaspespu_put);
	pthread_create(&pth1, NULL, pthrtn, &ctx1);
	pthread_create(&pth2, NULL, pthrtn, &ctx2);

	/* 1. Getting the ls address of the memory area where the put will be */
	/*    executed (via mailboxes)                                        */
	while(!spe_out_mbox_status(ctx1));
	spe_out_mbox_read(ctx1, &shared_lsaddr.h, 1);
	while(!spe_out_mbox_status(ctx1));
	spe_out_mbox_read(ctx1, &shared_lsaddr.l, 1);

	/* 2. constructing the effective address adding the base address of   */
	/*    LS area of the "get" process to the LS address just received    */
	ls_base_addr.e = 0;    /* assigning 0 to make sure garbage is removed */
	ls_base_addr.p = spe_ls_area_get(ctx1);
	shared_ea.e = shared_lsaddr.e + ls_base_addr.e;
	
	/* 3. Sending the calculated address to the "put" process.            */
	spe_in_mbox_write(ctx2, &shared_ea.h, 1, 1);
	spe_in_mbox_write(ctx2, &shared_ea.l, 1, 1);

	/* synchronizing with the put process for the termination of the dma  */
	/* transfer.                                                          */
	while(!spe_out_mbox_status(ctx2));
	spe_out_mbox_read(ctx1, &bell, 1);

	/* Waking up the "get" process                                        */
	spe_in_mbox_write(ctx1, &bell, 1, 1);

	/* Joining the threads                                                */
	pthread_join(pth1, NULL);
	pthread_join(pth2, NULL);
	spe_context_destroy(ctx1);
	spe_context_destroy(ctx2);
	return 0;

}
	

Code snippet: process PGET, SPE side

/* =============================================================================
 *										
 *       Filename:  dmaspespu_get.c						
 *										
 *    Description:  waits for transfer and prints the value.			
 *										
 *         Author:  Alessandro Piras (laynor@gmail.com)				
 *        Company:								
 *										
 * ===========================================================================*/

#include <stdio.h>
#include <spu_intrinsics.h>
#include <spu_mfcio.h>
#include "../common.h"

#define MYALIGN(vardecl, n) vardecl\\
	__attribute__ ((__aligned__ (n)))



MYALIGN(volatile unsigned int cane, 128);
int main(unsigned long long spe_id, unsigned long long argp, unsigned long long envp)
{
	union eat cane_addr;
	cane = 200;
	/* spu_dsync();                                                   */
	/* I'm not sure if it's indeed useful/required, it worked without */

	cane_addr.e=0;     /* assigning 0 to make sure garbage is removed */

	cane_addr.p = &cane;
	/* Sending the address of the memory area where the other spe     */
	/* process will transfer its data                                 */
	spu_write_out_mbox(cane_addr.h);
	spu_write_out_mbox(cane_addr.l);
	/* Synchronizing with the ppe                                     */
	spu_read_in_mbox();
	printf("(spu1) value: %u\
", cane);
	return 0;
}

Code snippet: process PPUT, SPE side

/* =============================================================================
 *										
 *       Filename:  dmaspespu_put.c						
 *										
 *    Description:  issues a dma on the LS of spe1.				
 *										
 *         Author:  Alessandro Piras (laynor@gmail.com)				
 *        Company:								
 *										
 * ===========================================================================*/

#include <stdio.h>
#include <spu_intrinsics.h>
#include <spu_mfcio.h>
#include <commonlib.h>
#include "../common.h"	

MYALIGN(volatile unsigned int x, 128);
int main(unsigned long long spe_id, unsigned long long argp, 
	unsigned long long envp)
{
	union eat shared_ea;
	/* Getting the target LS area effective address from ppe              */
	shared_ea.h = spu_read_in_mbox();
	shared_ea.l = spu_read_in_mbox();
	
	x = 210;
	/* Starting the dma trasnfer                                          */
	mfc_put((volatile void *)&x, shared_ea.e, 
		sizeof(unsigned int), 31, 0, 0);
	/* Waiting for the dma transfer to finish                             */
	mfc_write_tag_mask(1<<31);
	mfc_read_tag_status_all();
	printf("value = %d\
", x);
	/* Synchronizing with the ppe                                         */
	spu_write_out_mbox(10);
	return 0;
}