标签：string err 单机 mapReduce filename 简易版 intermediate func os

import "fmt"
import "6.824/mr"
import "plugin"
import "os"
import "log"
import "io/ioutil"
import "sort"

// for sorting by key.
type ByKey []mr.KeyValue

// for sorting by key.
func (a ByKey) Len() int           {  return len(a) }
func (a ByKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByKey) Less(i, j int) bool {  return a[i].Key < a[j].Key }

func main() {
    if len(os.Args) < 3 {
        fmt.Fprintf(os.Stderr,  "Usage: mrsequential xxx.so inputfiles...\n" )
        os.Exit(1)
    }

    mapf, reducef := loadPlugin(os.Args[1])

    //
    // read each input file,
    // pass it to Map,
    // accumulate the intermediate Map output.
    //
    intermediate := []mr.KeyValue{}
    for _, filename :=  range os.Args[2:] {
        file, err := os.Open(filename)
        if err != nil {
            log.Fatalf( "cannot open %v" , filename)
        }
        content, err := ioutil.ReadAll(file)
        if err != nil {
            log.Fatalf( "cannot read %v" , filename)
        }
        file.Close()
        kva := mapf(filename, string(content))
        intermediate = append(intermediate, kva...)
    }

    //
    // a big difference from real MapReduce is that all the
    // intermediate data is in one place, intermediate[],
    // rather than being partitioned into NxM buckets.
    //

    sort.Sort(ByKey(intermediate))

    oname :=  "mr-out-0"
    ofile, _ := os.Create(oname)

    //
    // call Reduce on each distinct key in intermediate[],
    // and print the result to mr-out-0.
    //
    i := 0
    for i < len(intermediate) {
        j := i + 1
        for j < len(intermediate) && intermediate[j].Key == intermediate[i].Key {
            j++
        }
        values := []string{}
        for k := i; k < j; k++ {
            values = append(values, intermediate[k].Value)
        }
        output := reducef(intermediate[i].Key, values)

        // this is the correct format for each line of Reduce output.
        fmt.Fprintf(ofile,  "%v %v\n" , intermediate[i].Key, output)

        i = j
    }

    ofile.Close()
}

//
// load the application Map and Reduce functions
// from a plugin file, e.g. ../mrapps/wc.so
//
func loadPlugin(filename string) ( func (string, string) []mr.KeyValue,  func (string, []string) string) {
    p, err := plugin.Open(filename)
    if err != nil {
        log.Fatalf( "cannot load plugin %v" , filename)
    }
    xmapf, err := p.Lookup( "Map" )
    if err != nil {
        log.Fatalf( "cannot find Map in %v" , filename)
    }
    mapf := xmapf.( func (string, string) []mr.KeyValue)
    xreducef, err := p.Lookup( "Reduce" )
    if err != nil {
        log.Fatalf( "cannot find Reduce in %v" , filename)
    }
    reducef := xreducef.( func (string, []string) string)

    return mapf, reducef
}

摘自 MIT6.824

标签：string,err,单机,mapReduce,filename,简易版,intermediate,func,os
来源： https://www.cnblogs.com/thotf/p/16413164.html

本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享；
2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关；
3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关；
4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除；
5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。