Source file src/pkg/encoding/csv/reader.go

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package csv reads and writes comma-separated values (CSV) files.
     6	//
     7	// A csv file contains zero or more records of one or more fields per record.
     8	// Each record is separated by the newline character. The final record may
     9	// optionally be followed by a newline character.
    10	//
    11	//	field1,field2,field3
    12	//
    13	// White space is considered part of a field.
    14	//
    15	// Carriage returns before newline characters are silently removed.
    16	//
    17	// Blank lines are ignored.  A line with only whitespace characters (excluding
    18	// the ending newline character) is not considered a blank line.
    19	//
    20	// Fields which start and stop with the quote character " are called
    21	// quoted-fields.  The beginning and ending quote are not part of the
    22	// field.
    23	//
    24	// The source:
    25	//
    26	//	normal string,"quoted-field"
    27	//
    28	// results in the fields
    29	//
    30	//	{`normal string`, `quoted-field`}
    31	//
    32	// Within a quoted-field a quote character followed by a second quote
    33	// character is considered a single quote.
    34	//
    35	//	"the ""word"" is true","a ""quoted-field"""
    36	//
    37	// results in
    38	//
    39	//	{`the "word" is true`, `a "quoted-field"`}
    40	//
    41	// Newlines and commas may be included in a quoted-field
    42	//
    43	//	"Multi-line
    44	//	field","comma is ,"
    45	//
    46	// results in
    47	//
    48	//	{`Multi-line
    49	//	field`, `comma is ,`}
    50	package csv
    51	
    52	import (
    53		"bufio"
    54		"bytes"
    55		"errors"
    56		"fmt"
    57		"io"
    58		"unicode"
    59	)
    60	
    61	// A ParseError is returned for parsing errors.
    62	// The first line is 1.  The first column is 0.
    63	type ParseError struct {
    64		Line   int   // Line where the error occurred
    65		Column int   // Column (rune index) where the error occurred
    66		Err    error // The actual error
    67	}
    68	
    69	func (e *ParseError) Error() string {
    70		return fmt.Sprintf("line %d, column %d: %s", e.Line, e.Column, e.Err)
    71	}
    72	
    73	// These are the errors that can be returned in ParseError.Error
    74	var (
    75		ErrTrailingComma = errors.New("extra delimiter at end of line")
    76		ErrBareQuote     = errors.New("bare \" in non-quoted-field")
    77		ErrQuote         = errors.New("extraneous \" in field")
    78		ErrFieldCount    = errors.New("wrong number of fields in line")
    79	)
    80	
    81	// A Reader reads records from a CSV-encoded file.
    82	//
    83	// As returned by NewReader, a Reader expects input conforming to RFC 4180.
    84	// The exported fields can be changed to customize the details before the
    85	// first call to Read or ReadAll.
    86	//
    87	// Comma is the field delimiter.  It defaults to ','.
    88	//
    89	// Comment, if not 0, is the comment character. Lines beginning with the
    90	// Comment character are ignored.
    91	//
    92	// If FieldsPerRecord is positive, Read requires each record to
    93	// have the given number of fields.  If FieldsPerRecord is 0, Read sets it to
    94	// the number of fields in the first record, so that future records must
    95	// have the same field count.  If FieldsPerRecord is negative, no check is
    96	// made and records may have a variable number of fields.
    97	//
    98	// If LazyQuotes is true, a quote may appear in an unquoted field and a
    99	// non-doubled quote may appear in a quoted field.
   100	//
   101	// If TrailingComma is true, the last field may be an unquoted empty field.
   102	//
   103	// If TrimLeadingSpace is true, leading white space in a field is ignored.
   104	type Reader struct {
   105		Comma            rune // Field delimiter (set to ',' by NewReader)
   106		Comment          rune // Comment character for start of line
   107		FieldsPerRecord  int  // Number of expected fields per record
   108		LazyQuotes       bool // Allow lazy quotes
   109		TrailingComma    bool // Allow trailing comma
   110		TrimLeadingSpace bool // Trim leading space
   111		line             int
   112		column           int
   113		r                *bufio.Reader
   114		field            bytes.Buffer
   115	}
   116	
   117	// NewReader returns a new Reader that reads from r.
   118	func NewReader(r io.Reader) *Reader {
   119		return &Reader{
   120			Comma: ',',
   121			r:     bufio.NewReader(r),
   122		}
   123	}
   124	
   125	// error creates a new ParseError based on err.
   126	func (r *Reader) error(err error) error {
   127		return &ParseError{
   128			Line:   r.line,
   129			Column: r.column,
   130			Err:    err,
   131		}
   132	}
   133	
   134	// Read reads one record from r.  The record is a slice of strings with each
   135	// string representing one field.
   136	func (r *Reader) Read() (record []string, err error) {
   137		for {
   138			record, err = r.parseRecord()
   139			if record != nil {
   140				break
   141			}
   142			if err != nil {
   143				return nil, err
   144			}
   145		}
   146	
   147		if r.FieldsPerRecord > 0 {
   148			if len(record) != r.FieldsPerRecord {
   149				r.column = 0 // report at start of record
   150				return record, r.error(ErrFieldCount)
   151			}
   152		} else if r.FieldsPerRecord == 0 {
   153			r.FieldsPerRecord = len(record)
   154		}
   155		return record, nil
   156	}
   157	
   158	// ReadAll reads all the remaining records from r.
   159	// Each record is a slice of fields.
   160	// A successful call returns err == nil, not err == EOF. Because ReadAll is
   161	// defined to read until EOF, it does not treat end of file as an error to be
   162	// reported.
   163	func (r *Reader) ReadAll() (records [][]string, err error) {
   164		for {
   165			record, err := r.Read()
   166			if err == io.EOF {
   167				return records, nil
   168			}
   169			if err != nil {
   170				return nil, err
   171			}
   172			records = append(records, record)
   173		}
   174		panic("unreachable")
   175	}
   176	
   177	// readRune reads one rune from r, folding \r\n to \n and keeping track
   178	// of how far into the line we have read.  r.column will point to the start
   179	// of this rune, not the end of this rune.
   180	func (r *Reader) readRune() (rune, error) {
   181		r1, _, err := r.r.ReadRune()
   182	
   183		// Handle \r\n here.  We make the simplifying assumption that
   184		// anytime \r is followed by \n that it can be folded to \n.
   185		// We will not detect files which contain both \r\n and bare \n.
   186		if r1 == '\r' {
   187			r1, _, err = r.r.ReadRune()
   188			if err == nil {
   189				if r1 != '\n' {
   190					r.r.UnreadRune()
   191					r1 = '\r'
   192				}
   193			}
   194		}
   195		r.column++
   196		return r1, err
   197	}
   198	
   199	// unreadRune puts the last rune read from r back.
   200	func (r *Reader) unreadRune() {
   201		r.r.UnreadRune()
   202		r.column--
   203	}
   204	
   205	// skip reads runes up to and including the rune delim or until error.
   206	func (r *Reader) skip(delim rune) error {
   207		for {
   208			r1, err := r.readRune()
   209			if err != nil {
   210				return err
   211			}
   212			if r1 == delim {
   213				return nil
   214			}
   215		}
   216		panic("unreachable")
   217	}
   218	
   219	// parseRecord reads and parses a single csv record from r.
   220	func (r *Reader) parseRecord() (fields []string, err error) {
   221		// Each record starts on a new line.  We increment our line
   222		// number (lines start at 1, not 0) and set column to -1
   223		// so as we increment in readRune it points to the character we read.
   224		r.line++
   225		r.column = -1
   226	
   227		// Peek at the first rune.  If it is an error we are done.
   228		// If we are support comments and it is the comment character
   229		// then skip to the end of line.
   230	
   231		r1, _, err := r.r.ReadRune()
   232		if err != nil {
   233			return nil, err
   234		}
   235	
   236		if r.Comment != 0 && r1 == r.Comment {
   237			return nil, r.skip('\n')
   238		}
   239		r.r.UnreadRune()
   240	
   241		// At this point we have at least one field.
   242		for {
   243			haveField, delim, err := r.parseField()
   244			if haveField {
   245				fields = append(fields, r.field.String())
   246			}
   247			if delim == '\n' || err == io.EOF {
   248				return fields, err
   249			} else if err != nil {
   250				return nil, err
   251			}
   252		}
   253		panic("unreachable")
   254	}
   255	
   256	// parseField parses the next field in the record.  The read field is
   257	// located in r.field.  Delim is the first character not part of the field
   258	// (r.Comma or '\n').
   259	func (r *Reader) parseField() (haveField bool, delim rune, err error) {
   260		r.field.Reset()
   261	
   262		r1, err := r.readRune()
   263		if err != nil {
   264			// If we have EOF and are not at the start of a line
   265			// then we return the empty field.  We have already
   266			// checked for trailing commas if needed.
   267			if err == io.EOF && r.column != 0 {
   268				return true, 0, err
   269			}
   270			return false, 0, err
   271		}
   272	
   273		if r.TrimLeadingSpace {
   274			for r1 != '\n' && unicode.IsSpace(r1) {
   275				r1, err = r.readRune()
   276				if err != nil {
   277					return false, 0, err
   278				}
   279			}
   280		}
   281	
   282		switch r1 {
   283		case r.Comma:
   284			// will check below
   285	
   286		case '\n':
   287			// We are a trailing empty field or a blank line
   288			if r.column == 0 {
   289				return false, r1, nil
   290			}
   291			return true, r1, nil
   292	
   293		case '"':
   294			// quoted field
   295		Quoted:
   296			for {
   297				r1, err = r.readRune()
   298				if err != nil {
   299					if err == io.EOF {
   300						if r.LazyQuotes {
   301							return true, 0, err
   302						}
   303						return false, 0, r.error(ErrQuote)
   304					}
   305					return false, 0, err
   306				}
   307				switch r1 {
   308				case '"':
   309					r1, err = r.readRune()
   310					if err != nil || r1 == r.Comma {
   311						break Quoted
   312					}
   313					if r1 == '\n' {
   314						return true, r1, nil
   315					}
   316					if r1 != '"' {
   317						if !r.LazyQuotes {
   318							r.column--
   319							return false, 0, r.error(ErrQuote)
   320						}
   321						// accept the bare quote
   322						r.field.WriteRune('"')
   323					}
   324				case '\n':
   325					r.line++
   326					r.column = -1
   327				}
   328				r.field.WriteRune(r1)
   329			}
   330	
   331		default:
   332			// unquoted field
   333			for {
   334				r.field.WriteRune(r1)
   335				r1, err = r.readRune()
   336				if err != nil || r1 == r.Comma {
   337					break
   338				}
   339				if r1 == '\n' {
   340					return true, r1, nil
   341				}
   342				if !r.LazyQuotes && r1 == '"' {
   343					return false, 0, r.error(ErrBareQuote)
   344				}
   345			}
   346		}
   347	
   348		if err != nil {
   349			if err == io.EOF {
   350				return true, 0, err
   351			}
   352			return false, 0, err
   353		}
   354	
   355		if !r.TrailingComma {
   356			// We don't allow trailing commas.  See if we
   357			// are at the end of the line (being mindful
   358			// of trimming spaces).
   359			c := r.column
   360			r1, err = r.readRune()
   361			if r.TrimLeadingSpace {
   362				for r1 != '\n' && unicode.IsSpace(r1) {
   363					r1, err = r.readRune()
   364					if err != nil {
   365						break
   366					}
   367				}
   368			}
   369			if err == io.EOF || r1 == '\n' {
   370				r.column = c // report the comma
   371				return false, 0, r.error(ErrTrailingComma)
   372			}
   373			r.unreadRune()
   374		}
   375		return true, r1, nil
   376	}