/*
 * heat diffusion example, as discussed in textbook.
 *
 * command line arguments are number of points, maximum number of
 *   iterations, convergence threshold, and optional flag "values"
 *   to print final values.
 * timing information goes to stdout, final values to stderr.
 *
 * parallel version with MPI.
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <mpi.h>

#define LEFTVAL 1.0
#define RIGHTVAL 10.0

void parse_arguments(int argc, char *argv[], 
        int *nx, int *maxsteps, double *threshold, int *print_flag);
void initialize(double uk[], double ukp1[], int num_points, int num_procs, 
        int myID);
void print_values(double uk[], int num_points, int myID);
int global_index(int local_index, int num_points, int myID);

int main(int argc, char *argv[]) {

    int nx;
    int maxsteps;
    double threshold;
    int print_flag;

    double *uk;
    double *ukp1;
    double *temp;
    double dx, dt;
    double start_time, end_time;
    double maxdiff, maxdiff_local, diff;
    int step, i;

    int num_procs, myID, left_nbr, right_nbr;
    int num_points, loop_start, loop_end;
    MPI_Status status; 

    /* MPI initialization */
    MPI_Init(&argc, &argv);
    MPI_Comm_size (MPI_COMM_WORLD, &num_procs);
    MPI_Comm_rank(MPI_COMM_WORLD, &myID);

    parse_arguments(argc, argv, &nx, &maxsteps, &threshold, &print_flag);

    MPI_Barrier(MPI_COMM_WORLD);
    start_time = MPI_Wtime();

    dx = 1.0/nx;
    dt = 0.5*dx*dx;
    maxdiff = threshold;

    if ((nx % num_procs) != 0) {
        fprintf(stderr, "Number of processes must evenly divide %d\n", nx);
        MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
    }

    left_nbr = myID - 1; /* ID of left "neighbor" process */
    right_nbr = myID + 1; /* ID of right "neighbor" process */
    num_points = (nx / num_procs);
    /* uk, ukp1 include a "ghost cell" at each end */
    uk = malloc(sizeof(double) * (num_points+2));
    ukp1 = malloc(sizeof(double) * (num_points+2));
    if (!uk || !ukp1) {
        fprintf(stderr, "Unable to allocate memory\n");
        MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
    }

    initialize(uk, ukp1, num_points, num_procs, myID);

    for (step = 0; (step < maxsteps) && (maxdiff >= threshold); ++step) {

        /* exchange boundary information */
        if (myID != 0) 
            MPI_Send(&uk[1], 1, MPI_DOUBLE, left_nbr, 0, MPI_COMM_WORLD);
        if (myID != num_procs-1)
            MPI_Send(&uk[num_points], 1, MPI_DOUBLE, right_nbr, 0, MPI_COMM_WORLD);
        if (myID != 0)
            MPI_Recv(&uk[0], 1, MPI_DOUBLE, left_nbr, 0, MPI_COMM_WORLD, &status);
        if (myID != num_procs-1)
            MPI_Recv(&uk[num_points+1],1, MPI_DOUBLE, right_nbr, 0, MPI_COMM_WORLD,
                    &status);

        /* compute new values points */
        loop_start = (myID == 0) ? 2 : 1;
        loop_end = (myID == num_procs-1) ? num_points-1 : num_points;
        for (i = loop_start; i <= loop_end; ++i) {
          ukp1[i]=uk[i]+ (dt/(dx*dx))*(uk[i+1]-2*uk[i]+uk[i-1]);
        }
        
        /* check for convergence */
        maxdiff_local = 0.0;
        for (i = 1; i <= num_points; ++i) {
            diff = fabs(uk[i] - ukp1[i]);
            if (diff > maxdiff_local) maxdiff_local = diff;
        }
        MPI_Allreduce(&maxdiff_local, &maxdiff, 1, MPI_DOUBLE, MPI_MAX,
                MPI_COMM_WORLD);

        /* "copy" ukp1 to uk by swapping pointers */
        temp = ukp1; ukp1 = uk; uk = temp;
    }

    MPI_Barrier(MPI_COMM_WORLD); /* sloppy -- to get more meaningful timing */
    end_time = MPI_Wtime();

    if (print_flag) {
        print_values(uk, num_points, myID);
    }

    if (myID == 0) {
        printf("MPI program (%d processes):\n", num_procs);
        printf("nx = %d, maxsteps = %d, threshold = %g\n", 
                nx, maxsteps, threshold);
        if (maxdiff < threshold) {
            printf("converged in %d iterations\n", step);
        }
        else {
            printf("failed to converge in %d iterations, maxdiff = %g\n", 
                    step, maxdiff);
        }
        printf("execution time = %g\n", end_time - start_time);
    }
 
    /* clean up and end */
    MPI_Finalize();
    return EXIT_SUCCESS;
}

void parse_arguments(int argc, char *argv[], 
        int *nx, int *maxsteps, double *threshold, int *print_flag) {
    char *usage_msg = 
    "usage is %s points max_iterations convergence_threshold [\"values\"]\n";

    if (argc < 4) {
        fprintf(stderr, usage_msg, argv[0]);
        MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
    }
    *nx = 0;
    *maxsteps = 0;
    *threshold = 0.0;
    *nx = atoi(argv[1]);
    *maxsteps = atoi(argv[2]);
    *threshold = atof(argv[3]);
    *print_flag = 0;
    if ((*nx <= 0) || (*maxsteps <= 0) || (*threshold <= 0)) {
        fprintf(stderr, usage_msg, argv[0]);
        MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
    }
    if ((argc > 4) && (strcmp(argv[4], "values") == 0)) {
        *print_flag = 1;
    }
}

void initialize(double uk[], double ukp1[], int num_points, 
        int num_procs, int myID) {
    /* uk, ukp1 include "ghost cell" at each end */
    int i;
    for (i = 1; i <= num_points; ++i)
        uk[i] = 0.0;
    /* left endpoint */
    if (myID == 0)
        uk[1] = LEFTVAL;
    /* right endpoint */
    if (myID == num_procs-1)
        uk[num_points] = RIGHTVAL;
    /* copy values to ukp1 */
    for (i = 1; i <= num_points; ++i)
        ukp1[i] = uk[i];
} 

void print_values(double uk[], int num_points, int myID) {
    int i;
    int global_i;
    for (i = 1; i <= num_points; ++i) {
        global_i = global_index(i, num_points, myID);
        fprintf(stderr, "uk[%010d] = %14.10f\n", global_i, uk[i]);
    }
}

int global_index(int local_index, int num_points, int myID) {
    return local_index-1 + myID*num_points;
}